Update configs to define ICU78 C++ executor and datasets (#535)

* Update configs to define ICU78 C++ executor and datasets

* Update C++ executor and testgen/test data for ICU 78 (CLDR 48/Unicode 17)

Author: echeran

README.md

@@ -532,31 +532,40 @@ These are usually the first changes to be made because ICU4C includes updates to
 locale data, and test data for many components.
 
 1. Test Driver:
-* Add new ICU version data in several places in testdriver/datasets.py
-
-2. testgen:
-* Add a new directory for the icu version under testgen, e.g., icu76
-
-* In this directory, copy test data from sources including icu4c/source. These
-  files include collation tests, number format data, and others.
-
-!!! TODO: Add details on the sources.
-
-* Add new CLDR test data generated from CLDR sources (!!! details !!!)
-
-3. schema: Add any new parameters in test data sources to test schema files.
-
-4. Add a function in setup.sh to download the new ICU4C release.
-
-5. Update run_config.json to reference new versions of executors and tests to run
-
-### NodeJS and some data updates
-
-NodeJS is usually updated several weeks after an ICU public release. Check on
-the site [Node.js Releases](https://nodejs.org/en/about/previous-releases) for
-the latest versions of NodeJS. Under each entry, the "changelog" will indicate
-any updates to icu, e.g., [Version 23.3.0 on 2024-11-20]
-(https://github.com/nodejs/node/blob/main/doc/changelogs/CHANGELOG_V23.md#23.3.0) which includes ICU76.1.
+    * Add new ICU version data in several places in testdriver/datasets.py
+1. testgen:
+    * Add a new directory for the icu version under testgen, e.g., icu76
+    * In this directory, copy test data from sources including icu4c/source. These
+files include collation tests, number format data, and others.
+        * `test/testdata/CollationTest_NON_IGNORABLE_SHORT.txt`
+        * `test/testdata/CollationTest_SHIFTED_SHORT.txt`
+        * `test/testdata/collationtest.txt`
+        * `test/testdata/dcfmtest.txt`
+        * `test/testdata/numberformattestspecification.txt`
+        * `test/testdata/numberpermutationtest.txt`
+    * Add new CLDR test data generated from CLDR sources
+        * `common/testData/localeIdentifiers/likelySubtags.txt`
+        * `common/testData/localeIdentifiers/localeDisplayName.txt`
+        * `common/supplemental/ordinals.xml`
+        * `common/supplemental/pluralRanges.xml`
+        * `common/supplemental/plurals.xml`
+        * `common/testData/datetime/datetime.json`
+    * Create `message_fmt2` test files
+        * Create subdir `message_fmt2`
+        * Copy the Readme file from `icu76/README.md` (or modify)
+        * Copy the test files from the MessageFormat 2 repo as described by the Readme into a subdirectory within `message_fmt2` called `message-format-wg-tests`
+1. schema: Add any new parameters in test data sources to test schema files.
+1. Add a function in setup.sh to download the new ICU4C release.
+1. Update run_config.json to reference new versions of executors and tests to run
+1. NodeJS and some data updates
+    * Ex: edit in `relativedatetime_fmt.py` the variable (potentially local to some function) called `icu_nvm_versions` that defines the version of NodeJS being used for the ICU version (version of data).
+    The exact version of NodeJS that has been released with the version of ICU can be found on the [NodeJS version pages](https://nodejs.org/en/about/previous-releases). \
+    \
+    NodeJS is usually updated several weeks after an ICU public release. Check on
+    the site [NodeJS Releases](https://nodejs.org/en/about/previous-releases) for
+    the latest versions of NodeJS. Under each entry, the "changelog" will indicate
+    any updates to icu, e.g., [Version 23.3.0 on 2024-11-20]
+    (https://github.com/nodejs/node/blob/main/doc/changelogs/CHANGELOG_V23.md#23.3.0) which includes ICU76.1.
 
 #### Add references in testdriver/datasets.py
 

run_config.json

@@ -23,6 +23,30 @@
       "per_execution": 10000
     }
   },
+  {
+    "prereq": {
+      "name": "Get ICU4C 78",
+      "version": "78.1",
+      "command": "bash ../executors/cpp/set_icu4c_binary.sh ../gh-cache/icu4c-78.1-Ubuntu22.04-x64.tgz"
+    },
+    "run": {
+      "icu_version": "icu78",
+      "exec": "cpp",
+      "test_type": [
+        "collation",
+        "datetime_fmt",
+        "lang_names",
+        "likely_subtags",
+        "list_fmt",
+        "message_fmt2",
+        "number_fmt",
+        "plural_rules",
+        "rdt_fmt",
+        "segmenter"
+      ],
+      "per_execution": 10000
+    }
+  },
   {
     "prereq": {
       "name": "Get ICU4C 76",

schema/schema_validator.py

@@ -456,7 +456,9 @@ def main(args):
     schema_validator.test_data_base = os.path.split(base_folders[0])[0]
     schema_validator.test_output_base = os.path.split(os.path.split(result_folders[0])[0])[0]
     schema_validator.icu_versions = ['icu71', 'icu72', 'icu73', 'icu74', 'icu75',
-                                     'icu76']
+                                     'icu76', 
+                                     'icu77', 'icu78'
+                                     ]
     schema_validator.executors = ['node', 'rust', 'dart_web', 'dart_native', 'icu4j']
 
     logger.info('Checking test outputs')

setup.sh

@@ -94,6 +94,13 @@ function download_77_1() {
   fi
 }
 
+function download_78_1() {
+  if [[ ! -f icu4c-78.1-Ubuntu22.04-x64.tgz ]]
+  then
+    wget https://github.com/unicode-org/icu/releases/download/release-78.1/icu4c-78.1-Ubuntu22.04-x64.tgz
+  fi
+}
+
  pushd gh-cache
 
  download_71_1
@@ -104,5 +111,6 @@ function download_77_1() {
  download_75_1
  download_76_1
  download_77_1
+ download_78_1
 
  popd

testdriver/datasets.py

@@ -52,6 +52,7 @@ class ICUVersion(Enum):
   ICU75 = "75"
   ICU76 = "76"
   ICU77 = "77"
+  ICU78 = "78"
 
 # TODO: Consider adding a trunk version for testing ICU / CLDR before
 # a complete release.
@@ -78,6 +79,7 @@ class CLDRVersion(Enum):
   CLDR45 = "45"
   CLDR46 = "46"
   CLDR47 = "47"
+  CLDR48 = "48"
 
 def latestCldrVersion():
   return CLDRVersion.CLDR43  # TODO: Fix this
@@ -99,6 +101,7 @@ def resolveCldr(text):
     CLDRVersion.CLDR45: [ICUVersion.ICU75],
     CLDRVersion.CLDR46: [ICUVersion.ICU76],
     CLDRVersion.CLDR47: [ICUVersion.ICU77],
+    CLDRVersion.CLDR48: [ICUVersion.ICU78],
 }
 
 # TODO: Can this be added to a configuration file?
@@ -437,6 +440,12 @@ def has(self, exec):
     CLDRVersion.CLDR47, versionICU=ICUVersion.ICU77,
     env={'LD_LIBRARY_PATH': '/tmp/icu/icu/usr/local/lib', 'PATH': '/tmp/icu77/bin'})
 
+allExecutors.addSystem(
+    system, CppVersion.Cpp,
+    '../executors/cpp/executor',
+    CLDRVersion.CLDR48, versionICU=ICUVersion.ICU78,
+    env={'LD_LIBRARY_PATH': '/tmp/icu/icu/usr/local/lib', 'PATH': '/tmp/icu78/bin'})
+
 system = 'newLanguage'
 allExecutors.addSystem(system, '0.1.0',
                        '/bin/newExecutor',

testgen/generators/relativedatetime_fmt.py

@@ -12,8 +12,9 @@ class RelativeDateTimeFmtGenerator(DataGenerator):
     json_verify = {"test_type": "rdt_fmt"}
 
     def process_test_data(self):
-        # Use NOde JS to create the .json files
+        # Use Node JS to create the .json files
         icu_nvm_versions = {
+            'icu78': '25.2.1',
             'icu77': '24.0.0',
             'icu76': '23.3.0',
             'icu75': '22.9.0',

testgen/icu77/message_fmt2/README.md

@@ -0,0 +1,9 @@
+# MessageFormat 2 tests
+
+The tests within the `message-format-wg-tests/` directory were copied from https://github.com/unicode-org/message-format-wg
+
+```sh
+curl -L https://github.com/unicode-org/message-format-wg/tarball/main | tar -xz \
+  && mv ./unicode-org-message-format-wg-*/test/tests ./message-format-wg-tests \
+  && rm -rf ./unicode-org-message-format-wg-*
+```