Datetime fixes. Updating known issues in date time implementations (#512)

* Updating number format data generation and NodeJS number_fmt

* Updated C++ number format, fixing errors and failing tests

* Fix number format in ICU4J

* Remove cout

* Add halfOdd to possible test results

* Add dateTimeFormatType to datagen, schema, and CPP version

* Updating regex for unsupported test types.

* Updates as suggested

* Classify some datetime_fmt errors as known issues, fixing some ICU4C and ICU4J failures.

* Handle Node known issue for timezone only format

* Responding to suggestions

* Remove unused variable

* Remove extra line

* Revise known issues for digit comparison

* Mark timezone only as unsupported in NodeJS datetime_fmt

* Simplified comparisons

Author: Craig Cornelius

executors/cpp/datetime_fmt.cpp

@@ -129,6 +129,11 @@ auto TestDatetimeFmt(json_object *json_in) -> string {
   // skeleton or date_style.
   string default_skeleton_string = "M/d/yyyy";
 
+  // Indicates if the library outputs date and time with the "at", as in "July 1 at 10:00"
+  bool supports_atTime = true;
+
+  string dateTimeFormatType_str = "";
+
   if (options_obj != nullptr) {
     json_object* option_item = json_object_object_get(options_obj, "dateStyle");
     if (option_item != nullptr) {
@@ -141,6 +146,31 @@ auto TestDatetimeFmt(json_object *json_in) -> string {
       timeStyle_str = json_object_get_string(option_item);
       time_style = StringToEStyle(timeStyle_str);
     }
+
+    option_item = json_object_object_get(options_obj, "dateTimeFormatType");
+    if (option_item != nullptr) {
+      // What this data item expects.
+      dateTimeFormatType_str = json_object_get_string(option_item);
+      // Check if this is not supported?
+      if ((dateTimeFormatType_str == "atTime" && !supports_atTime) ||
+          (dateTimeFormatType_str != "atTime" && supports_atTime)) {
+        // Inexact result is unsupported.
+        json_object_object_add(
+            return_json,
+            "error_type",
+            json_object_new_string("unsupported"));
+        json_object_object_add(
+            return_json,
+            "unsupported",
+            json_object_new_string("format type"));
+        string detail_str = "formatType: " + dateTimeFormatType_str;
+        json_object_object_add(
+            return_json,
+            "error_detail",
+            json_object_new_string(detail_str.c_str()));
+        return json_object_to_json_string(return_json);
+      }
+    }
   }
 
   json_object *date_skeleton_item =

executors/node/datetime_fmt.js

@@ -169,6 +169,16 @@ module.exports = {
       test_options['dateStyle'] = input_options['dateStyle'];
     }
 
+    if ('semanticSkeleton' in input_options) {
+      // Check for known issue when format output should give only the time zone.
+      if (input_options['semanticSkeleton'] == 'Z') {
+        return_json['error'] = 'unsupported';
+        return_json['error_detail'] = 'Requested timezone without date or time';
+        return_json['unsupported'] = 'timezone only';
+        return return_json;
+      }
+    }
+
     let calendar;
     try {
       calendar = input_options['calendar'];

schema/datetime_fmt/result_schema.json

@@ -35,6 +35,11 @@
              "and_more": {
                "description": "a placeholder for now",
                "type": "string"
+             },
+             "dateTimeFormatType": {
+               "description": "Indicates if the 'at' should be used, e.g., 'July 1 at 10:00",
+               "type": "string",
+               "enum": ["standard", "atTime"]
              }
            }
          },

schema/datetime_fmt/test_schema.json

@@ -141,7 +141,12 @@
              "semanticSkeletonLength": {
                "description": "Size of semantic skeleton for date/time format",
                "type": "string"
-             }
+             },
+             "dateTimeFormatType": {
+               "description": "Indicates if the 'at' should be used, e.g., 'July 1 at 10:00",
+               "type": "string",
+               "enum": ["standard", "atTime"]
+             }             
            }
          }
        },

testgen/generators/datetime_fmt.py

@@ -76,6 +76,9 @@ def generate_datetime_data_from_cldr(self, dt_json_path, run_limit=-1):
                     options['dateStyle'] = test_item['dateLength']
                 if 'timeLength' in test_item:
                     options['timeStyle'] = test_item['timeLength']
+                # Specifies if the expected output includes "at"
+                if 'dateTimeFormatType' in test_item:
+                    options['dateTimeFormatType'] = test_item['dateTimeFormatType']
 
                 if 'calendar' in test_item:
                     options['calendar'] = test_item['calendar']
@@ -92,8 +95,6 @@ def generate_datetime_data_from_cldr(self, dt_json_path, run_limit=-1):
                 input_string = u_time.isoformat().replace('+00:00', 'Z')
                 tz_offset_secs = raw_time.utcoffset().total_seconds()
 
-                if 'dateTimeFormatType' in test_item:
-                    options['dateTimeFormatType'] = test_item['dateTimeFormatType']
                 if 'classicalSkeleton' in test_item:
                     options['skeleton'] = test_item['classicalSkeleton']
                 if 'semanticSkeleton' in test_item:

verifier/check_known_issues.py

@@ -45,8 +45,12 @@
 # Global KnownIssue Info types and strings
 class knownIssueType(Enum):
     known_issue_nbsp_sp = 'ASCII Space instead of NBSP'
+    known_issue_sp_nbsp = 'NBSP instead of ASCII Space'
+
     known_issue_replaced_numerals = 'Not creating non-ASCII numerals'
 
+    known_issue_different_number_system = 'Different number systems'
+
     # Relative Date Time Format
     known_issue_unsupported_unit = 'Unsupported unit'  # https://github.com/unicode-org/conformance/issues/274
 
@@ -55,6 +59,9 @@ class knownIssueType(Enum):
     datetime_fmt_arabic_comma = 'Arabic comma vs. ASCII comma'
     datetime_unexpected_comma = 'Unexpected comma'
 
+    datetime_semantic_Z = 'NodeJS always includes date or time'
+
+
     # Likely Subtags
     likely_subtags_sr_latn = "sr_latin becoming en"
 
@@ -93,6 +100,19 @@ def diff_nbsp_vs_ascii_space(actual, expected_value):
     else:
         return None
 
+def diff_ascii_space_vs_nbsp(actual, expected_value):
+    # Returns the ID of this if the only difference in the two strings
+    # is Narrow Non-breaking Space (NBSP) in expected vs. ASCII space in the actual result.
+    # Found in datetime testing.
+    if not expected_value or not actual:
+        return None
+
+    # If replacing all the NBSP characdters in expected gives the actual result,
+    # then the only differences were with this type of space in formatted output.
+    if expected_value.replace(SP, NBSP) == actual:
+        return knownIssueType.known_issue_sp_nbsp
+    else:
+        return None
 
 def numerals_replaced_by_another_numbering_system(expected, actual):
     # If the only difference are one type of digit
@@ -114,28 +134,24 @@ def numerals_replaced_by_another_numbering_system(expected, actual):
     # Tag indicates the type of change.
     # i1:i2 is the range of the substring in expected
     # j1:j2 is the range of the substring in actual
+    different_number_systems = False
+    different_digit = False
+
 
     for diff in sm_opcodes:
         tag = diff[0]  # 'replace', 'delete', 'insert', or 'equal'
         old_val = expected[diff[1]:diff[2]]
         new_val = actual[diff[3]:diff[4]]
         if tag == 'replace':
             # expected[i1:i2] was replaced by actual[j1:j2]
-            if old_val.isdigit() and new_val.isdigit():
-                # TODO!! : check the value of the numeral
+            if old_val.isdigit() and new_val.isdigit() and len(old_val) == len(new_val):
                 # If the same value, then its a numbering system difference
-                if unicodedata.numeric(old_val) == unicodedata.numeric(new_val):
-                    digit_replace = True
-                else:
-                    # Both were digits but different numeric values
-                    non_digit_replacement = True
-            else:
-                # a digit was replaced with a non-digit
-                non_digit_replacement = True
-
-    # Only true if the only changes were replacing digits
-    if digit_replace and not non_digit_replace:
-        return knownIssueType.known_issue_replaced_numerals
+                for digit_old, digit_new in zip(old_val, new_val):
+                    if unicodedata.numeric(digit_old) == unicodedata.numeric(digit_new):
+                        different_number_systems = True
+
+    if different_number_systems:
+        return knownIssueType.known_issue_different_number_system
     else:
         return None
 
@@ -153,14 +169,20 @@ def dt_check_for_alternate_long_form(test, actual, expected):
         return None
     if actual.replace(' at', ',') == expected:
         return knownIssueType.datetime_fmt_at_inserted
+    # Thai language difference with "time" inserted
+    if actual.replace(' เวลา', '') == expected:
+        return knownIssueType.datetime_fmt_at_inserted
+    # Arabic
+    if actual.replace(' في', '،') == expected:
+        return knownIssueType.datetime_fmt_at_inserted
+
     return None
 
 
 def dt_check_arabic_comma(test, actual, expected):
     if expected.replace('\u002c', '\u060c') == actual:
         return knownIssueType.datetime_fmt_arabic_comma
-    else:
-         return None
+    return None
 
 
 def dt_unexpected_comma(test, actual,  expected):
@@ -177,15 +199,23 @@ def check_datetime_known_issues(test):
     try:
         result = test['result']
         expected = test['expected']
+        input_data = test.get('input_data')
+
         is_ki = diff_nbsp_vs_ascii_space(result, expected)
         if is_ki:
             # Mark the test with this issue
-            test['known_issue'] = knownIssueType.known_issue_nbsp_sp.value
+            test['known_issue'] = is_ki.value
+            remove_this_one = True
+
+        is_ki = diff_ascii_space_vs_nbsp(result, expected)
+        if is_ki:
+            # Mark the test with this issue
+            test['known_issue'] = is_ki.value
             remove_this_one = True
 
         is_ki = numerals_replaced_by_another_numbering_system(result, expected)
         if is_ki:
-            test['known_issue_id'] = knownIssueType.known_issue_replaced_numerals.value
+            test['known_issue_id'] = is_ki.value
             remove_this_one = True
 
         is_ki = dt_check_for_alternate_long_form(test, result, expected)
@@ -198,11 +228,17 @@ def check_datetime_known_issues(test):
             test['known_issue_id'] = is_ki.value
             remove_this_one = True
 
-        is_ki = dt_unexpectedcomma(test, result, expected)
+        is_ki = dt_unexpected_comma(test, result, expected)
         if is_ki:
             test['known_issue_id'] = is_ki.value
             remove_this_one = True
 
+        # Check if the semantic skeleton has "Z"
+        if input_data and 'semanticSkeleton' in input_data['options'] and \
+                input_data['options']['semanticSkeleton'] == 'Z':
+            test['known_issud_id'] = knownIssueType.datetime_semantic_Z.value
+            remove_this_one = True
+
     except BaseException as err:
         # Can't get the info
         pass