Add segmenter as test type for nodejs with minimal data (#458)

* Add segmenter as test type for nodejs with minimal data

* Update test and verify data

* Add segmenter test cases

* Updated generator to produce segmentation tests from NodeJS

* Updating data gen and characterizing differences in lists

* Remove temporary code

* Fix so segmenter data is recomputed

* Update as per comments on this PR

* Removing unneeded .gitignore items

* Add classification type

Author: Craig Cornelius

.gitignore

@@ -1,8 +1,11 @@
 #*#
 *~
 *#
+*.#
 .vscode
 
+*Minibuf-*
+    
 .pylintrc
 .idea
 .devcontainer
@@ -50,6 +53,3 @@ logrotate.state
 
 # Maven
 dependency-reduced-pom.xml
-
-# Dart
-executors/dart/.dart_tool/

executors/node/executor.js

@@ -34,6 +34,8 @@ let plural_rules = require('./plural_rules.js');
 
 let rdt_fmt = require('./relativedatetime_fmt.js');
 
+let segmenter = require('./segmenter.js');
+
 /**
  * TODOs:
  * 1. Handle other types of test cases.
@@ -64,7 +66,8 @@ const testTypes = {
   TestDisplayNames : Symbol("display_names"),
   TestListFmt : Symbol("list_fmt"),
   TestLocaleDisplayNames : Symbol("language_display_name"),
-  TestRelativeDateTimeFormat : Symbol("rdt_fmt")
+  TestRelativeDateTimeFormat : Symbol("rdt_fmt"),
+  TestSegmenter : Symbol("segmenter")
 };
 
 const supported_test_types = [
@@ -79,8 +82,9 @@ const supported_test_types = [
   Symbol("local_info"),
   Symbol("datetime_fmt"),
   Symbol("list_fmt"),
+  Symbol("plural_rules"),
   Symbol("rdt_fmt"),
-  Symbol("plural_rules")
+  Symbol("segmenter")
 ];
 
 const supported_tests_json = {
@@ -93,8 +97,9 @@ const supported_tests_json = {
     "lang_names",
     "language_display_name",
     "list_fmt",
+    "plural_rules",
     "rdt_fmt",
-    "plural_rules"
+    "segmenter"
   ]};
 
 // Test line-by-line input, with output as string.
@@ -166,7 +171,6 @@ function parseJsonForTestId(parsed) {
 let lineId = 0;
 rl.on('line', function(line) {
 
-  // if logging input.
   if (doLogInput > 0) {
     console.log("## NODE RECEIVED " + lineId + ' ' + line + ' !!!!!');
   }
@@ -247,6 +251,9 @@ rl.on('line', function(line) {
     } else
     if (test_type == "plural_rules") {
       outputLine = plural_rules.testPluralRules(parsedJson);
+    } else
+    if (test_type == "segmenter") {
+      outputLine = segmenter.testSegmenter(parsedJson);
     } else {
       outputLine = {'error': 'unknown test type',
                     'test_type': test_type,

executors/node/segmenter.js

@@ -0,0 +1,62 @@
+// Tests Intl segmenter
+
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Segmenter
+
+const supported_options = ['grapheme', 'word', 'sentence'];
+
+module.exports = {
+  testSegmenter: function (json) {
+    const label = json['label'];
+    const locale = json['locale'];
+    let options;
+    let ecma_intl_test_option;
+    if (json['options']) {
+      options = json['options'];
+      ecma_intl_test_option = options['granularity'];
+      if (options['granularity'] == 'grapheme_cluster') {
+        // Change to use ECMAScript's enum.
+        ecma_intl_test_option = 'grapheme';
+      }
+    }
+
+    let return_json = {'label': label};
+    if (!supported_options.includes(ecma_intl_test_option)) {
+      // Not supported
+      return_json['unsupported'] = 'granularity';
+      return_json['error_detail'] = ecma_intl_test_option;
+      return return_json;
+    }
+    let segmented_result = [];
+    try {
+      segmenter = new Intl.Segmenter(locale, {'granularity': ecma_intl_test_option});
+    } catch (error) {
+      /* Something is wrong with the constructor */
+      return_json['error'] = 'CONSTRUCTOR: ' + error.message;
+      return return_json;
+    }
+
+    let input;
+    try {
+      input = json['input'];
+    } catch (error) {
+      return_json['error'] = 'INPUT ERROR: ' + error.message;
+      return return_json;
+    }
+
+    try {
+      // Iterate through the results until error
+      const iterator = segmenter.segment(input)[Symbol.iterator]();
+      let seg_item = iterator.next();
+      while (! seg_item.done) {
+        segmented_result.push(seg_item.value.segment);
+        seg_item = iterator.next();
+      }
+    } catch (error) {
+      return_json['unsupported'] = 'SEGMENTER UNKNOWN ERROR: ' + error.message;
+    }
+    return_json['result'] = segmented_result;
+    return return_json;
+
+
+  }
+}

run_config.json

@@ -173,7 +173,8 @@
         "lang_names",
         "likely_subtags",
         "rdt_fmt",
-        "plural_rules"
+        "plural_rules",
+        "segmenter"
       ],
       "per_execution": 10000
     }

schema/schema_files.py

@@ -8,7 +8,8 @@
                   'message_fmt2',
                   'number_format',
                   'plural_rules',
-                  'rdt_fmt'
+                  'rdt_fmt',
+                  'segmenter'
                   ]
 
 TEST_FILE_TO_TEST_TYPE_MAP = {
@@ -26,7 +27,8 @@
     'plural_rules_test_file': 'plural_rules',
     'plural_rules_test': 'plural_rules',
     'rdt_fmt_test_file': 'rdt_fmt',
-    'rdt_fmt_test': 'rdt_fmt'
+    'rdt_fmt_test': 'rdt_fmt',
+    'segmenter_test': 'segmenter'
 }
 
 SCHEMA_FILE_MAP = {
@@ -107,6 +109,7 @@
             "prod_file": "lang_name_test_file.json"
         }
     },
+
     "likely_subtags": {
         "test_data": {
             "schema_file": "likely_subtags/test_schema.json",
@@ -121,6 +124,7 @@
             "prod_file": "likely_subtags_test.json"
         }
     },
+
     "list_fmt": {
         "test_data": {
             "schema_file": "list_fmt/test_schema.json",
@@ -135,6 +139,7 @@
             "prod_file": "list_fmt_test.json"
         }
     },
+
     "plural_rules": {
         "test_data": {
             "schema_file": "plural_rules/test_schema.json",
@@ -149,6 +154,7 @@
             "prod_file": "plural_rules_test.json"
         }
     },
+
     "message_fmt2": {
         "test_data": {
             "schema_file": "message_fmt2/test_schema.json",
@@ -163,6 +169,7 @@
             'prod_file': 'message_fmt2_test.json'
         }
     },
+
     "rdt_fmt": {
         "test_data": {
             "schema_file": "rdt_fmt/test_schema.json",
@@ -177,6 +184,21 @@
             "prod_file": "rdt_fmt_test.json"
         }
     },
+
+    "segmenter": {
+        "test_data": {
+            "schema_file": "segmenter/test_schema.json",
+            'prod_file': 'segmenter_test.json'
+        },
+        "verify_data": {
+            "schema_file": "segmenter/verify_schema.json",
+            'prod_file': 'segmenter2_verify.json'
+        },
+        "result_data": {
+            "schema_file": "segmenter/result_schema.json",
+            "prod_file": "segmenter_test.json"
+        }
+    },
     # Additional tests
 
 }

schema/segmenter/result_schema.json

@@ -0,0 +1,67 @@
+{"$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://github.com/unicode/conformance/segmenter_verify_schema.json'",
+ "title": "ICU Segmenter verify data description",
+ "description": "This documents of verify data for a segmenter",
+ "type": "object",
+ "properties": {
+   "additionalProperties": false,
+   "test_type": {
+     "description": "The name of the test",
+     "const": "segmenter"
+   },
+   "tests": {
+     "type": "array",
+     "items": {
+       "type": "object",
+       "additionalProperties": true,
+       "properties": {
+         "label": {
+           "description": "A numeric ID, unique for the set of tests",
+           "type": "string"
+         },
+         "verify": {
+           "description": "The expected value for the test result",
+           "type": "string"
+         },
+         "result": {
+           "description": "The actual result found",
+           "type": "array",
+           "items": {
+             "type": "string"
+           }
+           
+         },
+         "line" : {
+           "description": "line of the source of test data",
+           "type": "integer"
+         },
+         "error": {
+           "description": "What was unexpected",
+           "type": "string"
+         },
+         "error_message": {
+           "description": "More about the error",
+           "type": "string"
+         },
+         "actual_options": {
+           "description": "What was sent to the collation function",
+           "type": "string"
+         },
+         "input_data": {
+           "type": "string",
+           "description": "Information provided to the executor"
+         },
+         "actual_locale": {
+           "type": "string",
+           "description": "If present, the substitute locale actually used in the test"
+         }
+       }
+     },
+     "required": [
+       "label",
+       "result"
+     ]
+   }
+ },
+ "required": ["tests", "test_type"]
+}