Collation data generation fixes, reducing test failures (#472)

Craig Cornelius · web-flow · commit aec3bcb4e4cd · 2025-07-08T15:52:29.000-07:00
* Collation data generation fixes, reducing test failures

* Fixing some issues

* NodeJS collator fixed!

* fixing temporary workaround

* Fix regex

* NodeJS collation: change to 'best fit'

* Fix testgen for spaces in test data. Small improvement

* Updating code to add actual input when collation fails.

* Update collation result schema to handle actual options

* Fix problems with locale in nodejs collation.
diff --git a/executors/cpp/coll.cpp b/executors/cpp/coll.cpp
@@ -55,7 +55,7 @@ auto TestCollator(json_object *json_in) -> string {
   json_object *str2 = json_object_object_get(json_in, "s2");
 
   // Unescape the input strings?
-  string string1 = json_object_get_string(str1);
+  string string1 = json_object_get_string(str1) ;
   string string2 = json_object_get_string(str2);
 
   // Does this conversion preserve the data?
@@ -139,7 +139,7 @@ auto TestCollator(json_object *json_in) -> string {
     if (check_icu_error(status, return_json, "create RuleBasedCollator")) {
       // Put json_in as the actual input received.
       json_object_object_add(
-          return_json, "actual_input",
+          return_json, "actual_options",
           json_object_new_string(json_object_get_string(json_in)));
 
       return json_object_to_json_string(return_json);
@@ -233,15 +233,20 @@ auto TestCollator(json_object *json_in) -> string {
   if (!coll_result) {
     // Test did not succeed!
     // Include data compared in the failing test
+    json_object* actual_values = json_object_new_object();
+
     json_object_object_add(
-        return_json, "s1", json_object_new_string(string1.c_str()));
+        actual_values, "s1_actual", json_object_new_string(string1.c_str()));
     json_object_object_add(
-        return_json, "s2", json_object_new_string(string2.c_str()));
-
+        actual_values, "s2_actual", json_object_new_string(string2.c_str()));
     json_object_object_add(
-        return_json, "actual_input",
+        actual_values, "input",
         json_object_new_string(json_object_get_string(json_in)));
 
+    json_object_object_add(
+        return_json, "actual_options",
+        actual_values);
+
     // Record the actual returned value
     json_object_object_add(
         return_json, "compare", json_object_new_int64(uni_result));
diff --git a/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorTester.java b/executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/collator/CollatorTester.java
@@ -97,6 +97,25 @@ public ITestTypeOutputJson execute(ITestTypeInputJson inputJson) {
       }
     }
 
+    // Use the compare_type field to set the strength of collation test.
+    if (input.compare_type != null){
+      if (input.compare_type.equals("=")) {
+        coll.setStrength(Collator.IDENTICAL);
+      } else
+      if (input.compare_type.equals("<1")) {
+        coll.setStrength(Collator.PRIMARY);
+      } else
+      if (input.compare_type.equals("<2")) {
+        coll.setStrength(Collator.SECONDARY);
+      } else
+      if (input.compare_type.equals("<3")) {
+        coll.setStrength(Collator.TERTIARY);
+      } else
+      if (input.compare_type.equals("<4")) {
+        coll.setStrength(Collator.QUATERNARY);
+      }
+    }
+
     try {
       int collResult = coll.compare(input.s1, input.s2);
       // TODO! Use compare_type to check for <= or ==.
@@ -158,7 +177,7 @@ public Collator getCollatorForInput(CollatorInputJson input) {
         }
       }
     } else {
-      ULocale locale = ULocale.forLanguageTag(input.locale);
+      ULocale locale = new ULocale(input.locale);
       result = (RuleBasedCollator) Collator.getInstance(locale);
       if (input.rules != null) {
         String defaultRules = result.getRules();
diff --git a/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/collator/icu74/CollatorTest.java b/executors/icu4j/74/executor-icu4j/src/test/java/org/unicode/conformance/collator/icu74/CollatorTest.java
@@ -2,6 +2,7 @@
 
 import static org.junit.Assert.assertTrue;
 
+import org.junit.Ignore;
 import org.junit.Test;
 import org.unicode.conformance.testtype.collator.CollatorOutputJson;
 import org.unicode.conformance.testtype.collator.CollatorTester;
@@ -41,6 +42,54 @@ public void testAttributesAsArrayList() {
     assertTrue(output.result);
   }
 
+  @Test
+  public void testRule004() {
+    // in ICU 76.1 data
+    String testInput =
+        "{\"test_type\": \"collation\", \"compare_type\":\"&lt;3\",\"s1\":\"\u0002\",\"s2\":\"\u0300\",\"source_file\":\"collationtest.txt\",\"line\":43,\"label\":\"00002\",\"test_description\":\"simple CEs &amp; expansions\",\"rules\":\"&\\u0001<<<\\u0300&9<\\u0000&\\uA00A\\uA00B=\\uA002&\\uA00A\\uA00B\\u00050005=\\uA003\",\"hexhash\":\"7d3d23fab7f34c1cd44e90b40f7ed33c5bb317ba\"}";
+
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Test
+  public void test1362() {
+    // in ICU 76.1 data
+    String testInput =
+        "{\"test_type\": \"collation\", \"compare_type\":\"<2\",\"s1\":\"ae\",\"s2\":\"ä\",\"source_file\":\"collationtest.txt\",\"line\":2426,\"label\":\"01362\",\"localetag\":\"de-u-co-phonebk\", \"locale\": \"de@collation=PhoneBook\",\"test_description\":\"locale @collation=type should be case-insensitive\",\"hexhash\":\"893bdab906f7bc0e918ce388c7e78799d5913eaa\"}";
+
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+  @Ignore
+  @Test
+  public void test00144() {
+    // in ICU 76.1 data
+  String testInput =
+      "{\"test_type\": \"collation\", \"compare_type\":\"&lt;2\",\"s1\":\"cote\",\"s2\":\"cotÃ©\",\"source_file\":\"collationtest.txt\",\"line\":329,\"label\":\"00144\",\"locale\":\"root\",\"test_description\":\"côté with forwards secondary\",\"hexhash\":\"9a83942120095cac5793c15daebdf05cf30994ab\"}";
+
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
+
+@Ignore
+@Test
+  public void testIdentical() {
+    // In ICU 76.1 data
+    String testInput = "{\"test_type\": \"collation\",   \"compare_type\": \"=\",   \"s1\": \"a\u0327\",   \"s2\": \"\u00e2\u0093\u0090\u00e2\u009d\u00ba\",   \"source_file\": \"collationtest.txt\",   \"line\": 136,   \"label\": \"00032\",   \"test_description\": \"simple contractions\",   \"rules\": \"&a=\\u00e2\\u0093\\u0090&b<bz=\\u00e2\\u0093\\u0091&d<dz\\u0301=\\u00e2\\u0093\\u0093&z<a\\u0301=\\u00e2\\u0092\\u00b6<a\\u0301\\u0301=\\u00e2\\u0092\\u00b7<a\\u0301\\u0301\\u0358=\\u00e2\\u0092\\u00b8<a\\u030a=\\u00e2\\u0092\\u00b9<a\\u0323=\\u00e2\\u0092\\u00ba<a\\u0323\\u0358=\\u00e2\\u0092\\u00bb<a\\u0327\\u0323\\u030a=\\u00e2\\u0092\\u00bc<a\\u0327\\u0323bz=\\u00e2\\u0092\\u00bd&\\ud834\\udd58=\\u00e2\\u0081\\u00b0<\\ud834\\udd58\\ud834\\udd65=\\u00c2\\u00bc&\\u0001<<<\\ud834\\udd65=\\u00c2\\u00b9<<<\\ud834\\udd6d=\\u00c2\\u00b2<<<\\ud834\\udd65\\ud834\\udd6d=\\u00c2\\u00b3&\\u0301=\\u00e2\\u009d\\u00b6&\\u030a=\\u00e2\\u009d\\u00b7&\\u0308=\\u00e2\\u009d\\u00b8<<\\u0308\\u0301=\\u00e2\\u009d\\u00b9&\\u0327=\\u00e2\\u009d\\u00ba&\\u0323=\\u00e2\\u009d\\u00bb&\\u0331=\\u00e2\\u009d\\u00bc<<\\u0331\\u0358=\\u00e2\\u009d\\u00bd&\\u0334=\\u00e2\\u009d\\u00be&\\u0358=\\u00e2\\u009d\\u00bf&\\u0f71=\\u00e2\\u0091&\\u0f72=\\u00e2\\u0091\\u00a1&\\u0f73=\\u00e2\\u0091\\u00a2\",   \"hexhash\": \"b43889a20872ad4d242f9e94c942cc56cbb89b75\"}";
+
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
 /*  @Test
   public void testCompareLT2() {
     String testInput =
@@ -50,4 +99,15 @@ public void testCompareLT2() {
 
     assertTrue(output.result);
   }*/
+
+  @Test
+  public void test00002() {
+  String testInput =
+      "{\"test_type\": \"collation\", \"compare_type\":\"<3\",\"s1\":\"\u0002\",\"s2\":\"\u0300\",\"source_file\":\"collationtest.txt\",\"line\":43,\"label\":\"00002\",\"test_description\":\"simple CEs & expansions\",\"rules\":\"&\\\\u0001<<<\\u0300&9<\\u0000&\\uA00A\\uA00B=\\uA002&\\uA00A\\uA00B\\u00050005=\\uA003\",\"hexhash\":\"87be5cda089d675543eb91b948e2d7f74227ff0d\"}";
+
+    CollatorOutputJson output =
+        (CollatorOutputJson) CollatorTester.INSTANCE.getStructuredOutputFromInputStr(testInput);
+
+    assertTrue(output.result);
+  }
 }
diff --git a/executors/node/collator.js b/executors/node/collator.js
@@ -5,11 +5,52 @@ module.exports = {
   testCollationShort: function(json) {
     // Global default locale
 
+    let outputLine = {'label':json['label']};
+
     // Locale if provided in the test data.
-    let testLocale = undefined;
+    let testLocale = 'en';  // default
     if ('locale' in json) {
       testLocale = json['locale'];
     }
+
+    if (testLocale == 'root') {
+      outputLine =  {'label': json['label'],
+                     'error_message': "root locale",
+                     'unsupported': 'root locale',
+                     'error_detail': testLocale,
+                     'error': 'Unsupported locale'
+                    };
+      return outputLine;
+    }
+
+    // Check if this locale is actually supported
+    try {
+      const supported_locales =
+            Intl.Collator.supportedLocalesOf([testLocale], {localeMatcher: "best fit"});
+
+      if (supported_locales.length == 1 && supported_locales[0] != testLocale) {
+        testLocale = supported_locales[0];
+        outputLine['substituted_locale'] = testLocale;;
+      }
+      else if (supported_locales.length <= 0 ||
+               !supported_locales.includes(testLocale)) {
+        // Report as unsupported
+        outputLine['error_message'] = "unsupported locale";
+        outputLine['unsupported'] = testLocale;
+        outputLine['error_detail'] = supported_locales;
+        outputLine['error'] = "unsupported locale";
+        return outputLine;
+      }
+    } catch (error) {
+      console.log("ERROR @ 44 ", error.name, " ", error.message);
+      console.log(" testLocale = ", testLocale);
+      outputLine['unsupported'] = "supportedLocalsOf";
+      outputLine['error_message'] = error.message;
+      outputLine['error_detail'] = testLocale;
+      outputLine['error'] = error.name;
+      return outputLine;
+    }
+
     let testCollOptions = {};
     if ('ignorePunctuation' in json) {
       testCollOptions['ignorePunctuation'] = json['ignorePunctuation'];
@@ -30,11 +71,10 @@ module.exports = {
         testCollOptions['sensitivity'] = 'accent';
       } else
       if (strength == 'tertiary') {
-        testCollOptions['sensitivity'] = 'case';
+        testCollOptions['sensitivity'] = 'variant';
       }
     }
 
-    let outputLine = {'label':json['label']};
     // Get other fields if provided
     let rules = undefined;
     if ('rules' in json) {
@@ -86,27 +126,34 @@ module.exports = {
         outputLine['compare_result'] = compared;
       } else {
         // Additional info for the comparison
-        outputLine['actual_options'] = JSON.stringify(coll.resolvedOptions());
+        outputLine['actual_options'] = {
+          'compared_result': compared,
+          's1': d1,
+          's2': d2,
+          'options': JSON.stringify(coll.resolvedOptions())
+        };
         outputLine['compare_result'] = compared;
         outputLine['result'] = result;
       }
 
     } catch (error) {
       const error_message = error.message;
-      if (testLocale == "root" ||
-          error_message == "Incorrect locale information provided")  {
+      console.log('ERROR @ 135: ', error);
+      if (error_message == "Incorrect locale information provided")  {
         outputLine =  {'label': json['label'],
                        'error_message': error.message,
-                       'unsupported': 'root locale',
+                       'unsupported': 'UNSUPPORTED',
                        'error_detail': error_message + ': ' + testLocale,
-                       'error': 'Unsupported locale'
+                       'actual_options': JSON.stringify(coll.resolvedOptions()),
                       };
       } else {
+        console.log("ERROR @ 144 ", error.name, " ", error.message);
         // Another kind of error.
         outputLine =  {'label': json['label'],
                        'error_message': error.message,
                        'error_detail': testLocale,
-                       'error': error.name
+                       'error': error.name,
+                       'actual_options': JSON.stringify(coll.resolvedOptions()),
                       };
       }
     }
diff --git a/schema/collation/result_schema.json b/schema/collation/result_schema.json
@@ -84,8 +84,19 @@
            "type": "string"
          },
          "actual_options": {
+           "type": ["object", "string"],
+           "additionalProperties": true,
            "description": "Options used by collation as a string",
-           "type": "string"
+           "properties": {
+             "s1_actual": {
+               "type": "string",
+               "description": "actual input string1"
+             },
+             "s2_actual": {
+               "type": "string",
+               "description": "actual input string1"
+             }
+           }
          },
          "input_data": {
            "type": "string",
diff --git a/testdriver/testplan.py b/testdriver/testplan.py
@@ -470,10 +470,9 @@ def run_multitest_mode(self):
     def open_json_test_data(self):
         # Read JSON file with results.
         try:
-            input_file = open(self.inputFilePath,
-                              encoding='utf-8', mode='r')
-            file_raw = input_file.read()
-            input_file.close()
+            with open(self.inputFilePath,
+                      encoding='utf-8', mode='r') as input_file:
+                file_raw = input_file.read()
             try:
                 self.jsonData = json.loads(file_raw)
             except json.JSONDecodeError as error:
diff --git a/testgen/generators/base.py b/testgen/generators/base.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from abc import ABC, abstractmethod
+import codecs
 import copy
 import hashlib
 import json
@@ -82,9 +83,10 @@ def saveJsonFile(self, filename, data, indent=None):
                               filename)
 
         output_path = os.path.join(self.icu_version, filename)
-        output_file = open(output_path, "w", encoding="utf-8")
-        json.dump(data, output_file, indent=indent)
-        output_file.close()
+        # output_file = open(output_path, "w", encoding="utf8")
+
+        with open(output_path, "w", encoding="utf8") as output_file:
+            json.dump(data, output_file, indent=indent)
 
     def getTestDataFromGitHub(self, datafile_name, version):
         # Path for fetching test data from ICU repository
@@ -132,7 +134,7 @@ def readFile(self, filename, version="", filetype="txt"):
         if version:
             path = os.path.join(version, filename)
         try:
-            with open(path, "r", encoding="utf-8") as testdata:
+            with codecs.open(path, "r", encoding="utf-8") as testdata:
                 return json.load(testdata) if filetype == "json" else testdata.read()
         except BaseException as err:
             logging.warning("** readFile: %s", err)
diff --git a/testgen/generators/collation.py b/testgen/generators/collation.py
