ICU-23056 Integrate CLDR 48 release alpha0, part 4, rework ICU4C eraNames loading

- Now that eraNames data are in map form and EraRules has been updated, rework the
  ICU4C eraNames code so the CalendarSink treats the eraNamese data as a table,
  then a new initEras function converts to array and fills in missing values from
  parent(s). This solves the problem with not loading the narrow modern era names
  in Japanese, for example.
- Then we can remove the logKnownIssue test skips for ICU-23182/ICU-23186
- Also update tests for fixed Meiji start date

Author: DraganBesevic

icu4c/source/i18n/dtfmtsym.cpp

@@ -39,6 +39,7 @@
 #include "cmemory.h"
 #include "cstring.h"
 #include "charstr.h"
+#include "erarules.h"
 #include "dt_impl.h"
 #include "locbased.h"
 #include "gregoimp.h"
@@ -1776,50 +1777,8 @@ struct CalendarDataSink : public ResourceSink {
                 arraySizes.puti(path, dataArraySize, errorCode);
                 if (U_FAILURE(errorCode)) { return; }
             } else if (value.getType() == URES_TABLE) {
-                // We might have an eras table that is replacing an eras leaf array
-                if (path.startsWith(u"eras", 4)) {
-                    // path is one of eras/wide, eras/abbreviated, eras/narrow
-                    ResourceTable rDataTable = value.getTable(errorCode);
-                    int32_t dataTableSize = rDataTable.getSize();
-                    UVector dataList(uprv_deleteUObject, uhash_compareUnicodeString, dataTableSize, errorCode);
-                    if (U_FAILURE(errorCode)) { return; }
-                    // Expand the UVector as necessary to have index from 0 up to the max
-                    // eraCode, and fill in the slots for the eras defined in the resource data
-                    // (filling in empty strings for other eras as we expand, since they would
-                    // otherwise not get set to anything in particular such as null).
-                    for (int32_t dataTableIndex = 0; dataTableIndex < dataTableSize; dataTableIndex++) {
-                        rDataTable.getKeyAndValue(dataTableIndex, key, value);
-                        int32_t listIndex = uprv_strtol(key, nullptr, 10);
-                        if (listIndex + 1 > dataList.size()) {
-                            dataList.ensureCapacity(listIndex + 1, errorCode); // needed only to minimize expansions
-                            if (U_FAILURE(errorCode)) { return; }
-                            // Fill in empty strings for all added slots (else they are undefined)
-                            while (dataList.size() < listIndex + 1) {
-                                LocalPointer<UnicodeString> emptyString(new UnicodeString(), errorCode);
-                                dataList.adoptElement(emptyString.orphan(), errorCode);
-                            }
-                        }
-                        // Now set the eraName that we just read
-                        LocalPointer<UnicodeString> eraName((value.getType() == URES_STRING) ?
-                                new UnicodeString(value.getUnicodeString(errorCode)) : new UnicodeString(), errorCode);
-                        if (U_FAILURE(errorCode)) { return; }
-                        dataList.setElementAt(eraName.orphan(), listIndex);
-                    }
-                    // Now convert to array running from era code 0 to the max era we have data for, fill
-                    // in from the UVector
-                    int32_t dataArraySize = dataList.size();
-                    LocalArray<UnicodeString> dataArray(new UnicodeString[dataArraySize], errorCode);
-                    // Fill out dataArray from dataList (dataList.toArray did not seem to work as expected)
-                    for (int32_t dataArrayIndex = 0; dataArrayIndex < dataArraySize; dataArrayIndex++) {
-                        dataArray[dataArrayIndex] = std::move(*(reinterpret_cast<UnicodeString *>(dataList.elementAt(dataArrayIndex))));
-                    }
-                    // Save array...
-                    arrays.put(path, dataArray.orphan(), errorCode);
-                    arraySizes.puti(path, dataArraySize, errorCode);
-                } else {
-                    // We are not on a leaf, recursively process the subtable.
-                    processResource(path, key, value, errorCode);
-                }
+                // We are not on a leaf, recursively process the subtable.
+                processResource(path, key, value, errorCode);
                 if (U_FAILURE(errorCode)) { return; }
             }
 
@@ -1944,6 +1903,51 @@ initField(UnicodeString **field, int32_t& length, CalendarDataSink &sink, CharSt
     }
 }
 
+static void
+initEras(UnicodeString **field, int32_t& length, CalendarDataSink &sink, CharString &key, const UResourceBundle *ctebPtr, const char* eraWidth, int32_t maxEra, UErrorCode &status) {
+    if (U_SUCCESS(status)) {
+        length = 0;
+        UnicodeString keyUString(key.data(), -1, US_INV);
+        Hashtable *eraNamesTable = static_cast<Hashtable*>(sink.maps.get(keyUString));
+
+        if (eraNamesTable != nullptr) {
+            UErrorCode resStatus = U_ZERO_ERROR;
+            LocalUResourceBundlePointer ctewb(ures_getByKeyWithFallback(ctebPtr, eraWidth, nullptr, &resStatus));
+            const UResourceBundle *ctewbPtr = (U_SUCCESS(resStatus))? ctewb.getAlias() : nullptr;
+            *field = new UnicodeString[maxEra + 1];
+            if (*field == nullptr) {
+                status = U_MEMORY_ALLOCATION_ERROR;
+                return;
+            }
+            length = maxEra + 1;
+            for (int32_t eraCode = 0; eraCode <= maxEra; eraCode++) {
+                char eraCodeStr[12]; // T_CString_integerToString is documented to generate at most 12 bytes including nul terminator
+                int32_t eraCodeStrLen = T_CString_integerToString(eraCodeStr, eraCode, 10);
+                UnicodeString eraCodeKey = UnicodeString(eraCodeStr, eraCodeStrLen, US_INV);
+                UnicodeString *eraName = static_cast<UnicodeString*>(eraNamesTable->get(eraCodeKey));
+                (*field)[eraCode].remove();
+                if (eraName != nullptr) {
+                    // Get eraName from map (created by CalendarSink)
+                    (*field)[eraCode].fastCopyFrom(*eraName);
+                } else if (ctewbPtr != nullptr) {
+                    // Try filling in missing items from parent locale(s)
+                    resStatus = U_ZERO_ERROR;
+                    LocalUResourceBundlePointer ctewkb(ures_getByKeyWithFallback(ctewbPtr, eraCodeStr, nullptr, &resStatus));
+                    if (U_SUCCESS(resStatus)) {
+                        int32_t eraNameLen;
+                        const UChar* eraNamePtr = ures_getString(ctewkb.getAlias(), &eraNameLen, &resStatus);
+                        if (U_SUCCESS(resStatus)) {
+                            (*field)[eraCode].setTo(false, eraNamePtr, eraNameLen);
+                        }
+                    }
+                }
+            }
+            return;
+        }
+        status = U_MISSING_RESOURCE_ERROR;
+    }
+}
+
 static void
 initLeapMonthPattern(UnicodeString *field, int32_t index, CalendarDataSink &sink, CharString &path, UErrorCode &status) {
     field[index].remove();
@@ -2344,17 +2348,30 @@ DateFormatSymbols::initializeData(const Locale& locale, const char *type, UError
     validLocale = Locale(ures_getLocaleByType(cb.getAlias(), ULOC_VALID_LOCALE, &status));
     actualLocale = Locale(ures_getLocaleByType(cb.getAlias(), ULOC_ACTUAL_LOCALE, &status));
 
+    // Era setup
+    if (type == nullptr) {
+        type = "gregorian";
+    }
+    LocalPointer<EraRules> eraRules(EraRules::createInstance(type, false, status));
+    int32_t maxEra = (U_SUCCESS(status))? eraRules->getMaxEraCode(): 0;
+    UErrorCode resStatus = U_ZERO_ERROR;
+    LocalUResourceBundlePointer ctpb(ures_getByKeyWithFallback(cb.getAlias(), type, nullptr, &resStatus));
+    LocalUResourceBundlePointer cteb(ures_getByKeyWithFallback(ctpb.getAlias(), gErasTag, nullptr, &resStatus));
+    const UResourceBundle *ctebPtr = (U_SUCCESS(resStatus))? cteb.getAlias() : nullptr;
     // Load eras
-    initField(&fEras, fErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesAbbrTag, status), status);
+    initEras(&fEras, fErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesAbbrTag, status),
+            ctebPtr, gNamesAbbrTag, maxEra, status);
     UErrorCode oldStatus = status;
-    initField(&fEraNames, fEraNamesCount, calendarSink, buildResourcePath(path, gErasTag, gNamesWideTag, status), status);
+    initEras(&fEraNames, fEraNamesCount, calendarSink, buildResourcePath(path, gErasTag, gNamesWideTag, status),
+            ctebPtr, gNamesWideTag, maxEra, status);
     if (status == U_MISSING_RESOURCE_ERROR) { // Workaround because eras/wide was omitted from CLDR 1.3
         status = oldStatus;
         assignArray(fEraNames, fEraNamesCount, fEras, fErasCount);
     }
     // current ICU4J falls back to abbreviated if narrow eras are missing, so we will too
     oldStatus = status;
-    initField(&fNarrowEras, fNarrowErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesNarrowTag, status), status);
+    initEras(&fNarrowEras, fNarrowErasCount, calendarSink, buildResourcePath(path, gErasTag, gNamesNarrowTag, status),
+            ctebPtr, gNamesNarrowTag, maxEra, status);
     if (status == U_MISSING_RESOURCE_ERROR) { // Workaround because eras/wide was omitted from CLDR 1.3
         status = oldStatus;
         assignArray(fNarrowEras, fNarrowErasCount, fEras, fErasCount);

icu4c/source/test/intltest/dtifmtts.cpp

@@ -1251,11 +1251,6 @@ void DateIntervalFormatTest::expect(const char** data, int32_t data_length) {
         FieldPosition pos(FieldPosition::DONT_CARE);
         dtitvfmt->format(&dtitv, str.remove(), pos, ec);
         if (!assertSuccess("format in expect", ec)) return;
-        if (strcmp(locName, "ja-u-ca-japanese") == 0 &&
-            logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-            i++;
-            continue;
-        }
 
         if (strcmp(locName, "de") == 0 && 
             (oneSkeleton == UnicodeString(u"hv",-1) || oneSkeleton == UnicodeString(u"hz",-1)) &&
@@ -2335,10 +2330,6 @@ void DateIntervalFormatTest::testTicket21222ROCEraDiff() {
 void DateIntervalFormatTest::testTicket21222JapaneseEraDiff() {
     IcuTestErrorCode status(*this, "testTicket21222JapaneseEraDiff");
 
-    if (logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-        return;
-    }
-
     LocalPointer<Calendar> cal(Calendar::createInstance(*TimeZone::getGMT(), status));
     if (U_FAILURE(status)) {
         errln("Failure encountered: %s", u_errorName(status));

icu4c/source/test/intltest/dtptngts.cpp

@@ -696,11 +696,6 @@ void IntlTestDateTimePatternGeneratorAPI::testAPI(/*char *par*/)
             resultDate = sdf.format(testDate, resultDate);
             if ( resultDate != patternResults[localeIndex][resultIndex] ) {
                 const auto* calendar = sdf.getCalendar();
-				if (localeIndex == 6 && (dataIndex == 1 || dataIndex == 3) &&
-                	logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                	resultIndex++;
-                  	continue;
-        		}                
                 errln(UnicodeString("\nERROR: Test various skeletons[") + (dataIndex-1) + UnicodeString("], localeIndex ") + localeIndex +
                       u". Got: \"" + resultDate +
                       u"\" with calendar " + calendar->getType() +

icu4c/source/test/intltest/incaltst.cpp

@@ -238,13 +238,11 @@ void IntlCalendarTest::quasiGregorianTest(Calendar& cal, const Locale& gcl, cons
             cal.get(UCAL_YEAR, status) + "/" +
             (cal.get(UCAL_MONTH, status) + 1) + "/" + cal.get(UCAL_DATE, status) + " (" + UnicodeString(cal.getType()) + ")");
     } else {
-      if (!logKnownIssue("ICU-23186", "Japanese calendar round trip fails")) {
         errln(UnicodeString("Fail: (millis to fields)") + D + " => " + cal.get(UCAL_ERA, status) + ":" +
             cal.get(UCAL_YEAR, status) + "/" +
             (cal.get(UCAL_MONTH, status)+1) + "/" + cal.get(UCAL_DATE, status) +
             ", expected " + era + ":" + year + "/" + (month+1) + "/" +
             dayOfMonth +  " (" + UnicodeString(cal.getType()));
-      }
     }
   }
   delete grego;
@@ -405,8 +403,8 @@ void IntlCalendarTest::TestJapanese() {
     // BE 2542 == 1999 CE
     int32_t data[] = { 
         //       Jera         Jyr  Gyear   m             d
-        JapaneseCalendar_MEIJI, 1, 1868, UCAL_SEPTEMBER, 8,
-        JapaneseCalendar_MEIJI, 1, 1868, UCAL_SEPTEMBER, 9,
+        JapaneseCalendar_MEIJI, 1, 1868, UCAL_OCTOBER, 23,
+        JapaneseCalendar_MEIJI, 1, 1868, UCAL_OCTOBER, 24,
         JapaneseCalendar_MEIJI, 2, 1869, UCAL_JUNE, 4,
         JapaneseCalendar_MEIJI, 45, 1912, UCAL_JULY, 29,
         JapaneseCalendar_TAISHO, 1, 1912, UCAL_JULY, 30,
@@ -553,7 +551,7 @@ void IntlCalendarTest::TestJapaneseFormat() {
 
     // Test parse with incomplete information
     SimpleDateFormat fmti(UnicodeString("G y"), Locale("en_US@calendar=japanese"), status);
-    aDate = -3197117222000.0;
+    aDate = -3193229222000.0;
     CHECK(status, "creating date format instance");
     str.remove();
     fmt2.format(aDate, str);
@@ -562,21 +560,17 @@ void IntlCalendarTest::TestJapaneseFormat() {
     fmti.format(aDate, str);
     logln(UnicodeString() + "as Japanese Calendar: " + str);
     expected = u"Meiji 1";
-    if(str != expected) {
-        if (!logKnownIssue("ICU-23186", "Japanese calendar round trip fails")) {
-            errln("Expected " + expected + " but got " + str);
-        }
+    if (str != expected) {
+        errln("Expected " + expected + " but got " + str);
     }
     otherDate = fmti.parse(expected, status);
-    if(otherDate != aDate) { 
+    if (otherDate != aDate) { 
         UnicodeString str3;
         ParsePosition pp;
         fmti.parse(expected, *cal2, pp);
         fmti.format(otherDate, str3);
-        if (!logKnownIssue("ICU-23186", "Japanese calendar round trip fails")) {
-            errln("Parse incorrect of " + expected + " - wanted " + aDate + " but got " +  " = " +
-                otherDate + ", " + str3 + " = " + CalendarTest::calToStr(*cal2) );
-        }
+        errln("Parse incorrect of " + expected + " - wanted " + aDate + " but got " +  " = " +
+            otherDate + ", " + str3 + " = " + CalendarTest::calToStr(*cal2) );
     } else {
         logln("Parsed OK: " + expected);
     }
@@ -745,18 +739,14 @@ void IntlCalendarTest::TestForceGannenNumbering()
         }
         testString2 = testFmt2->format(refDate, testString2);
         if (testString2.length() < 2 || testString2.charAt(1) != 0x0031) {
-            if (!logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                errln(UnicodeString("Formatting year 1 in created numeric style, got " + testString2 + " but expected 2nd char to be 1"));
-            }
+            errln(UnicodeString("Formatting year 1 in created numeric style, got " + testString2 + " but expected 2nd char to be 1"));
         }
         // Now switch the patterns and verify that Gannen use follows the pattern
         testFmt1->applyPattern(patNumr);
         testString1.remove();
         testString1 = testFmt1->format(refDate, testString1);
         if (testString1.length() < 2 || testString1.charAt(1) != 0x0031) {
-            if (!logKnownIssue("ICU-23182", "Japanese calendar formatting")) {
-                errln(UnicodeString("Formatting year 1 in applied numeric style, got " + testString1 + " but expected 2nd char to be 1"));
-            }
+            errln(UnicodeString("Formatting year 1 in applied numeric style, got " + testString1 + " but expected 2nd char to be 1"));
         }
         testFmt2->applyPattern(patText);
         testString2.remove();