CLDR-18956 cldr-json: RBNF new format (#5060)

Author: srl295

docs/site/downloads/cldr-48.md

@@ -19,7 +19,7 @@ adapting software to the conventions of different languages.
 
 CLDR 48 was an open submission cycle allowing contributors to supply data for their languages via the CLDR Survey Tool —
 data that is widely used to support much of the world’s software.
-This data is also a factor in determining which languages are supported on mobile phones and computer operating systems. 
+This data is also a factor in determining which languages are supported on mobile phones and computer operating systems.
 
 ### Changes
 
@@ -225,7 +225,38 @@ For a full listing, see [Transforms Delta].
 
 ### JSON Data Changes
 
-- TBD
+- RBNF
+  - Just as with the RBNF data format change in XML [CLDR-8909], the JSON data also has a change in structure. [CLDR-18956].
+  - Below is an example of the changed data format.
+  - The new data item is the `_rbnfRulesFile` key. Its value is the name of a data file in the same directory, containing the raw rules.  (Note: Do not interpret the .txt file’s name in any way.)
+  - The previous data format is included for this release, but will be removed in a future release. In this case, the `%digits-ordinal` (and any other such keys) will be removed.
+
+
+```js
+{
+    "rbnf": {
+      "OrdinalRules": {
+        "%digits-ordinal": [
+          [
+            "-x",
+            "−→→;"
+          ],
+          [
+            "0",
+            "=#,##0=;"
+          ]
+        ],
+        "_rbnfRulesFile": "ar-OrdinalRules.txt"
+      },
+    }
+}
+```
+
+The `ar-OrdinalRules.txt` file contains all rules for this locale:
+
+   %digits-ordinal:
+   -x: −>>;
+   0: =#,##0=;
 
 ### File Changes
 The following files are new in the release:
@@ -258,7 +289,7 @@ The following files are new in the release:
 ### V49 advance warnings
 The following changes are planned for CLDR 49. Please plan accordingly to avoid disruption.
 - The default week numbering changes to ISO instead being based on the calendar week starting in CLDR 48 [CLDR-18275]. The calendar week will be more clearly targeted at matching usage in displayed month calendars.
-- The pre-Meiji Japanese eras will be removed: There was too much uncertainty in the exact values 
+- The pre-Meiji Japanese eras will be removed: There was too much uncertainty in the exact values
 and feedback that the general practice for exact dates is to use Gregorian for pre-Meiji dates.
 - The major components in [supplementalData.xml](https://github.com/unicode-org/cldr/blob/main/common/supplemental/supplementalData.xml) and [supplementalMetadata.xml](https://github.com/unicode-org/cldr/blob/main/common/supplemental/supplementalMetadata.xml) files are slated to be organized more logically and moved into separate files.
     - This will make it easier for implementations to filter out data that they don't need, and make internal maintenance easier. This will not affect the data: just which file it is located in. Please plan to update XML and JSON parsers accordingly.
@@ -281,6 +312,8 @@ in particular, see [Exhibit 1](https://unicode.org/copyright.html#Exhibit1).
 For web pages with different views of CLDR data, see [http://cldr.unicode.org/index/charts](/index/charts).
 
 [CLDR-5708]: https://unicode-org.atlassian.net/browse/CLDR-5708
+[CLDR-8909]: https://unicode-org.atlassian.net/browse/CLDR-8909
+[CLDR-11400]: https://unicode-org.atlassian.net/browse/CLDR-11400
 [CLDR-14479]: https://unicode-org.atlassian.net/browse/CLDR-14479
 [CLDR-16004]: https://unicode-org.atlassian.net/browse/CLDR-16004
 [CLDR-16715]: https://unicode-org.atlassian.net/browse/CLDR-16715
@@ -290,7 +323,8 @@ For web pages with different views of CLDR data, see [http://cldr.unicode.org/in
 [CLDR-18219]: https://unicode-org.atlassian.net/browse/CLDR-18219
 [CLDR-18275]: https://unicode-org.atlassian.net/browse/CLDR-18275
 [CLDR-18311]: https://unicode-org.atlassian.net/browse/CLDR-18311
-[CLDR-11400]: https://unicode-org.atlassian.net/browse/CLDR-11400
+[CLDR-18956]: https://unicode-org.atlassian.net/browse/CLDR-18956
+
 [Delta DTDs]: https://unicode.org/cldr/charts/48/supplemental/dtd_deltas.html
 [BCP47 Delta]: https://unicode.org/cldr/charts/48/delta/bcp47.html
 [Supplemental Delta]: https://unicode.org/cldr/charts/48/delta/supplemental-data.html

tools/cldr-code/src/main/java/org/unicode/cldr/json/Ldml2JsonConverter.java

@@ -66,6 +66,7 @@
 import org.unicode.cldr.util.PatternCache;
 import org.unicode.cldr.util.StandardCodes;
 import org.unicode.cldr.util.SupplementalDataInfo;
+import org.unicode.cldr.util.SupplementalDataInfo.RBNFGroup;
 import org.unicode.cldr.util.Timer;
 import org.unicode.cldr.util.XMLSource;
 import org.unicode.cldr.util.XPathParts;
@@ -90,7 +91,7 @@ public class Ldml2JsonConverter {
     private static final String CLDR_PKG_PREFIX = "cldr-";
     private static final String FULL_TIER_SUFFIX = "-full";
     private static final String MODERN_TIER_SUFFIX = "-modern";
-    private static final String TRANSFORM_RAW_SUFFIX = ".txt";
+    private static final String EXTERNAL_RAW_SUFFIX = ".txt";
     private static Logger logger = Logger.getLogger(Ldml2JsonConverter.class.getName());
 
     enum RunType {
@@ -870,13 +871,45 @@ private int convertCldrItems(
                         }
                         if (type == RunType.rbnf) {
                             if (item.getFullPath().contains("/rbnfRules")) {
-                                System.err.println("TODO CLDR-18956: skipping NEW rules.");
-                                continue;
-                            }
-                            try {
-                                item.adjustRbnfPath();
-                            } catch (Throwable t) {
-                                throw new RuntimeException(location + ": " + item.getPath(), t);
+                                XPathParts fullPath =
+                                        XPathParts.getFrozenInstance(item.getFullPath())
+                                                .cloneAsThawed();
+                                XPathParts dpath =
+                                        XPathParts.getFrozenInstance(item.getPath())
+                                                .cloneAsThawed();
+                                final String type = fullPath.getAttributeValue(-2, "type");
+                                if (type == null)
+                                    throw new RuntimeException(
+                                            location
+                                                    + " Could not get rulesetGrouping type for "
+                                                    + fullPath.toString());
+                                // here, write the raw data
+                                final String rawTransformFile =
+                                        filename + "-" + type + EXTERNAL_RAW_SUFFIX;
+                                try (PrintWriter outf =
+                                        FileUtilities.openUTF8Writer(outputDir, rawTransformFile)) {
+                                    outf.println(item.getValue().trim());
+                                    // note: not logging the write here- it will be logged when the
+                                    // .json file is written.
+                                }
+                                dpath.setElement(-1, "ruleset");
+                                dpath.addElement("rulesFile");
+                                item.setPath(dpath.toString());
+                                // we have to do this because we want the rules file entry to sort
+                                // with the others.
+                                fullPath.setElement(-1, "ruleset");
+                                fullPath.setAttribute("ruleset", "_q", "-1");
+                                fullPath.setAttribute("ruleset", "type", "rulesFile");
+                                fullPath.addElement("rulesFile");
+                                item.setFullPath(fullPath.toString());
+                                // the value is now the raw filename
+                                item.setValue(rawTransformFile);
+                            } else {
+                                try {
+                                    item.adjustRbnfPath();
+                                } catch (Throwable t) {
+                                    throw new RuntimeException(location + ": " + item.getPath(), t);
+                                }
                             }
                         }
 
@@ -905,7 +938,7 @@ private int convertCldrItems(
                         if (item.getUntransformedPath()
                                 .startsWith("//supplementalData/transforms")) {
                             // here, write the raw data
-                            final String rawTransformFile = filename + TRANSFORM_RAW_SUFFIX;
+                            final String rawTransformFile = filename + EXTERNAL_RAW_SUFFIX;
                             try (PrintWriter outf =
                                     FileUtilities.openUTF8Writer(outputDir, rawTransformFile)) {
                                 outf.println(item.getValue().trim());
@@ -975,37 +1008,16 @@ private int convertCldrItems(
                     }
 
                     resolveSortingItems(out, nodesForLastItem, sortingItems);
+
                     resolveArrayItems(out, nodesForLastItem, arrayItems);
+
+                    out = postProcessAfterResolveArrayItems(filename, out);
+
+                    // special processing for unit preferences
                     if (js.section.contains("unitPreferenceData")) {
                         outputUnitPreferenceData(js, theItems, out, nodesForLastItem);
                     }
 
-                    // Special processing for transforms.
-                    if (type == RunType.transforms) {
-                        final JsonObject jo = out.getAsJsonObject("transforms");
-                        if (jo == null || jo.isEmpty()) {
-                            throw new RuntimeException(
-                                    "Could not get transforms object in " + filename);
-                        }
-                        @SuppressWarnings("unchecked")
-                        final Entry<String, JsonElement>[] s = jo.entrySet().toArray(new Entry[0]);
-                        if (s == null || s.length != 1) {
-                            throw new RuntimeException(
-                                    "Could not get 1 subelement of transforms in " + filename);
-                        }
-                        // key doesn't matter.
-                        // move subitem up
-                        out = s[0].getValue().getAsJsonObject();
-                        final Entry<String, JsonElement>[] s2 =
-                                out.entrySet().toArray(new Entry[0]);
-                        if (s2 == null || s2.length != 1) {
-                            throw new RuntimeException(
-                                    "Could not get 1 sub-subelement of transforms in " + filename);
-                        }
-                        // move sub-subitem up.
-                        out = s2[0].getValue().getAsJsonObject();
-                    }
-
                     // write JSON
                     try (PrintWriter outf = FileUtilities.openUTF8Writer(outputDir, outFilename)) {
                         outf.println(gson.toJson(out));
@@ -1051,6 +1063,67 @@ private int convertCldrItems(
         return totalItemsInFile;
     }
 
+    private JsonObject postProcessAfterResolveArrayItems(String filename, JsonObject out) {
+        // Special processing for transforms.
+        if (type == RunType.transforms) {
+            final JsonObject jo = out.getAsJsonObject("transforms");
+            if (jo == null || jo.isEmpty()) {
+                throw new RuntimeException("Could not get transforms object in " + filename);
+            }
+            @SuppressWarnings("unchecked")
+            final Entry<String, JsonElement>[] s = jo.entrySet().toArray(new Entry[0]);
+            if (s == null || s.length != 1) {
+                throw new RuntimeException(
+                        "Could not get 1 subelement of transforms in " + filename);
+            }
+            // key doesn't matter.
+            // move subitem up
+            out = s[0].getValue().getAsJsonObject();
+            final Entry<String, JsonElement>[] s2 = out.entrySet().toArray(new Entry[0]);
+            if (s2 == null || s2.length != 1) {
+                throw new RuntimeException(
+                        "Could not get 1 sub-subelement of transforms in " + filename);
+            }
+            // move sub-subitem up.
+            out = s2[0].getValue().getAsJsonObject();
+        } else if (type == RunType.rbnf) {
+            // hoist all "ruleset" children
+            /*
+            From:
+                  "OrdinalRules": {
+                    "ruleset": [
+                        "ar-OrdinalRules.txt"
+                    ],
+                    "%digits-ordinal": [
+                    [
+                        …
+            To:
+                  "OrdinalRules": {
+                    "_rbnfRulesFile": "ar-OrdinalRules.txt",
+                    "%digits-ordinal": [
+                    [
+                        …
+             */
+
+            JsonObject o = out.getAsJsonObject("rbnf");
+            if (o == null) return out;
+            o = o.getAsJsonObject("rbnf");
+            if (o == null) return out;
+            for (final Entry<String, JsonElement> e : o.entrySet()) {
+                final String type = e.getKey();
+                if (RBNFGroup.valueOf(type) != null) {
+                    JsonObject ruleType = e.getValue().getAsJsonObject();
+                    if (ruleType == null) continue;
+                    JsonArray ruleset = ruleType.getAsJsonArray("ruleset");
+                    if (ruleset == null) continue;
+                    ruleType.addProperty("_rbnfRulesFile", ruleset.get(0).getAsString());
+                    ruleType.remove("ruleset");
+                }
+            }
+        }
+        return out;
+    }
+
     /**
      * Provide an opportunity to fix up the JsonObject before write, after items were added.
      *
@@ -1942,6 +2015,10 @@ private JsonArray outputStartArray(
         // Get the name of the parent of the array
         String objName = nodesInPath.get(arrayLevel - 1).getNodeKeyName();
         JsonArray array = new JsonArray();
+        if (!o.isJsonObject()) {
+            throw new IllegalStateException(
+                    "Not a JSON Object " + o + " when trying to add array " + objName + ": []");
+        }
         o.getAsJsonObject().add(objName, array);
 
         return array;
@@ -2136,10 +2213,16 @@ private void outputArrayItem(
             Map<String, String> attrAsValueMap = cldrNode.getAttrAsValueMap();
 
             if (attrAsValueMap.isEmpty()) {
-                JsonObject o = new JsonObject();
-                out.add(o);
-                o.addProperty(objName, value);
-            } else if (objName.equals("rbnfrule")) {
+                if (item.getUntransformedPath().startsWith("//ldml/rbnf/")
+                        && objName.equals("rulesFile")) {
+                    out.add(value);
+                } else {
+                    JsonObject o = new JsonObject();
+                    out.add(o);
+                    o.addProperty(objName, value);
+                }
+            } else if (item.getUntransformedPath().startsWith("//ldml/rbnf/")
+                    && objName.equals("rbnfrule")) {
                 writeRbnfLeafNode(out, item, attrAsValueMap);
             } else {
                 JsonObject o = new JsonObject();
@@ -2208,9 +2291,20 @@ private String attrToKey(String key) {
 
     private void writeRbnfLeafNode(
             JsonElement out, CldrItem item, Map<String, String> attrAsValueMap) throws IOException {
+        if (item.getUntransformedPath().contains("rbnfRules")) {
+            throw new RuntimeException(
+                    item.getUntransformedPath()
+                            + " = "
+                            + item.getFullPath()
+                            + " // "
+                            + attrAsValueMap.keySet().stream().collect(Collectors.joining(", ")));
+        }
         if (attrAsValueMap.size() != 1) {
             throw new IllegalArgumentException(
-                    "Error, attributes seem wrong for RBNF " + item.getUntransformedPath());
+                    "Error, attributes seem wrong for RBNF "
+                            + item.getUntransformedPath()
+                            + " = "
+                            + item.getFullPath());
         }
         Entry<String, String> entry = attrAsValueMap.entrySet().iterator().next();
         JsonArray arr = new JsonArray();