CLDR-16425 v43 BRS: JSON (#2749)

* CLDR-16425 v43 BRS: JSON

fixes for CoverageLevel reader
- use CalculatedCoverageLevels instead of homemade reader

* CLDR-16425 v43 BRS: JSON

- core package has everything, full has basic, modern has modern+
- add a -M option to include the modern package (on by default for now)
- add an effectiveCoverageLevels calculation for sublocales.  Also include this in the coverageLevels.json

* CLDR-16425 v43 BRS: fixes for parentLocales

Author: srl295

tools/cldr-code/src/main/java/org/unicode/cldr/json/Ldml2JsonConverter.java

@@ -1,6 +1,5 @@
 package org.unicode.cldr.json;
 
-import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -44,6 +43,7 @@
 import com.ibm.icu.number.NumberFormatter;
 import com.ibm.icu.number.Precision;
 import com.ibm.icu.util.NoUnit;
+import com.ibm.icu.util.ULocale;
 
 /**
  * Utility methods to extract data from CLDR repository and export it in JSON
@@ -61,7 +61,7 @@ public class Ldml2JsonConverter {
     enum RunType {
         all,
         main,
-        supplemental(false, false), // aka 'core'
+        supplemental(false, false), // aka 'cldr-core'
         segments, rbnf(false, true), annotations, annotationsDerived, bcp47(false, false);
 
         private final boolean isTiered;
@@ -146,7 +146,8 @@ private class AvailableLocales {
                 .add("identity", 'i', "(true|false)", "true",
                     "Whether to copy the identity info into all sections containing data")
                 .add("konfig", 'k', ".*", null, "LDML to JSON configuration file")
-                .add("pkgversion",  'V', ".*", getDefaultVersion(), "Version to be used in writing package files");
+                .add("pkgversion",  'V', ".*", getDefaultVersion(), "Version to be used in writing package files")
+                .add("Modern", 'M', "(true|false)", "true", "Whether to include the -modern tier");
 
     public static void main(String[] args) throws Exception {
         options.parse(args, true);
@@ -186,7 +187,8 @@ static void processType(final String runType) throws Exception {
             options.get("konfig").getValue(),
             options.get("pkgversion").getValue(),
             Boolean.parseBoolean(options.get("bcp47").getValue()),
-            Boolean.parseBoolean(options.get("bcp47-no-subtags").getValue())
+            Boolean.parseBoolean(options.get("bcp47-no-subtags").getValue()),
+            Boolean.parseBoolean(options.get("Modern").getValue())
         );
 
         DraftStatus status = DraftStatus.valueOf(options.get("draftstatus").getValue());
@@ -226,12 +228,14 @@ public int compareTo(JSONSection other) {
     private Set<String> packages;
     final private String pkgVersion;
     final private boolean strictBcp47;
+    final private boolean writeModernPackage;
     final private boolean skipBcp47LocalesWithSubtags;
     private LdmlConfigFileReader configFileReader;
 
     public Ldml2JsonConverter(String cldrDir, String outputDir, String runType, boolean fullNumbers, boolean resolve, String coverage, String match,
         boolean writePackages, String configFile, String pkgVersion,
-        boolean strictBcp47, boolean skipBcp47LocalesWithSubtags) {
+        boolean strictBcp47, boolean skipBcp47LocalesWithSubtags, boolean writeModernPackage) {
+        this.writeModernPackage = writeModernPackage;
         this.strictBcp47 = strictBcp47;
         this.skipBcp47LocalesWithSubtags = strictBcp47 && skipBcp47LocalesWithSubtags;
         this.cldrCommonDir = cldrDir;
@@ -432,7 +436,7 @@ private Map<JSONSection, List<CldrItem>> mapPathsToSections(AtomicInteger readCo
 
             // discard draft before transforming
             final String pathNoDraft = CLDRFile.DRAFT_PATTERN.matcher(path).replaceAll("");
-            final String fullPathNoDraft = CLDRFile.DRAFT_PATTERN.matcher(fullPath).replaceAll("");            
+            final String fullPathNoDraft = CLDRFile.DRAFT_PATTERN.matcher(fullPath).replaceAll("");
 
             final String pathNoXmlSpace = CLDRFile.XML_SPACE_PATTERN.matcher(pathNoDraft).replaceAll("");
             final String fullPathNoXmlSpace = CLDRFile.XML_SPACE_PATTERN.matcher(fullPathNoDraft).replaceAll("");
@@ -559,7 +563,7 @@ private int convertCldrItems(AtomicInteger readCount, int totalCount,
                             continue;
                         }
                         final boolean isModernTier = localeIsModernTier(filename);
-                        if (isModernTier) {
+                        if (isModernTier && writeModernPackage) {
                             tier = MODERN_TIER_SUFFIX;
                             if (type == RunType.main) {
                                 avl.modern.add(filenameAsLangTag);
@@ -598,9 +602,9 @@ private int convertCldrItems(AtomicInteger readCount, int totalCount,
                 List<String> outputDirs = new ArrayList<>();
                 outputDirs.add(outputDirname.toString());
                 if (writePackages && tier.equals(MODERN_TIER_SUFFIX) && js.packageName != null) {
-                    // if it is in 'modern', add it to 'full' also.
+                    // if it is in 'modern', add it to 'full' and core also.
                     outputDirs.add(outputDirname.toString().replaceFirst(MODERN_TIER_SUFFIX, FULL_TIER_SUFFIX));
-                    // Also need to make sure that the full package is added
+                    // Also need to make sure that the full and core package is added
                     packages.add(CLDR_PKG_PREFIX + js.packageName + FULL_TIER_SUFFIX);
                 }
 
@@ -746,12 +750,15 @@ private int convertCldrItems(AtomicInteger readCount, int totalCount,
     }
 
     private boolean localeIsModernTier(String filename) {
-        boolean isModernTier;
-        {
-            final Level localeCoverageLevel = sc.getHighestLocaleCoverageLevel("Cldr", filename);
-            isModernTier = (localeCoverageLevel.getLevel() >= Level.MODERN.getLevel()) || filename.equals(LocaleNames.ROOT) || filename.equals(LocaleNames.UND);
-        }
-        return isModernTier;
+        Level lev = CalculatedCoverageLevels.getInstance().getEffectiveCoverageLevel(filename);
+        if (lev == null) return false;
+        return lev.isAtLeast(Level.MODERN);
+    }
+
+    private boolean localeIsBasicTier(String filename) {
+        Level lev = CalculatedCoverageLevels.getInstance().getEffectiveCoverageLevel(filename);
+        if (lev == null) return false;
+        return lev.isAtLeast(Level.BASIC);
     }
 
     /**
@@ -965,11 +972,10 @@ public void writeReadme(String outputDir, String packageName) throws IOException
             outf.println(configFileReader.getPackageDescriptions().get(basePackageName));
             outf.println();
             if (packageName.endsWith(FULL_TIER_SUFFIX)) {
-                outf.println("This package contains the complete set of locales, including what is in the `" +
-                CLDR_PKG_PREFIX + basePackageName + MODERN_TIER_SUFFIX + "` package.");
+                outf.println("This package contains all locales.");
                 outf.println();
             } else if (packageName.endsWith(MODERN_TIER_SUFFIX)) {
-                outf.println("This package contains the set of locales listed as modern coverage. See also the `" +
+                outf.println("This package contains only the set of locales listed as modern coverage. See also the `" +
                 CLDR_PKG_PREFIX + basePackageName + FULL_TIER_SUFFIX + "` package.");
                 outf.println();
             }
@@ -1047,7 +1053,7 @@ public void writePackageJson(String outputDir, String packageName) throws IOExce
         final String basePackageName = getBasePackageName(packageName);
         String description = configFileReader.getPackageDescriptions().get(basePackageName);
         if (packageName.endsWith(FULL_TIER_SUFFIX)) {
-            description = description + " (complete)";
+            description = description + " (all locales)";
         } else if (packageName.endsWith(MODERN_TIER_SUFFIX)) {
             description = description + " (modern coverage locales)";
         }
@@ -1119,32 +1125,33 @@ public void writeDefaultContent(String outputDir) throws IOException {
 
     public void writeCoverageLevels(String outputDir) throws IOException {
         final Splitter SEMICOLON = Splitter.on(';').trimResults();
-        try (BufferedReader r = FileUtilities.openUTF8Reader(CLDRPaths.COMMON_DIRECTORY + "properties/", "coverageLevels.txt");
-            PrintWriter outf = FileUtilities.openUTF8Writer(outputDir + "/cldr-core", "coverageLevels.json");) {
-                final Map<String, String> covlocs = new TreeMap<>();
-                System.out.println("Creating packaging file => " + outputDir + "/cldr-core" + File.separator + "coverageLevels.json from coverageLevels.txt");
-                String line;
-                int no = 0;
-                while ((line = r.readLine()) != null) {
-                    no++;
-                    line = line.trim();
-                    if(line.startsWith("#") || line.isBlank()) {
-                        continue;
-                    }
-                    final List<String> l = SEMICOLON.splitToList(line);
-                    if (l.size() != 2) {
-                        throw new IllegalArgumentException("coverageLevels.txt:"+no+": expected 2 fields, got " + l.size());
-                    }
-                    final String uloc = l.get(0);
-                    final String level = l.get(1);
-                    final String bcp47loc = unicodeLocaleToString(uloc);
-                    if (covlocs.put(bcp47loc, level) != null) {
-                        throw new IllegalArgumentException("coverageLevels.txt:"+no+": duplicate locale " + bcp47loc);
-                    }
+        try (PrintWriter outf = FileUtilities.openUTF8Writer(outputDir + "/cldr-core", "coverageLevels.json");) {
+            final Map<String, String> covlocs = new TreeMap<>();
+            System.out.println("Creating packaging file => " + outputDir + "/cldr-core" + File.separator + "coverageLevels.json from coverageLevels.txt");
+            CalculatedCoverageLevels ccl = CalculatedCoverageLevels.getInstance();
+            for (final Map.Entry<String, org.unicode.cldr.util.Level> e : ccl.getLevels().entrySet()) {
+                final String uloc = e.getKey();
+                final String level = e.getValue().name().toLowerCase();
+                final String bcp47loc = unicodeLocaleToString(uloc);
+                if (covlocs.put(bcp47loc, level) != null) {
+                    throw new IllegalArgumentException("coverageLevels.txt: duplicate locale " + bcp47loc);
+                }
+            }
+            final Map<String, String> effectiveCovlocs = new TreeMap<>();
+            avl.full.forEach(loc -> {
+                final String uloc = ULocale.forLanguageTag(loc).toString();
+                final Level lev = ccl.getEffectiveCoverageLevel(uloc);
+                if (lev != null) {
+                    effectiveCovlocs.put(loc, lev.name().toLowerCase());
                 }
-                JsonObject obj = new JsonObject();
-                obj.add("coverageLevels", gson.toJsonTree(covlocs));
-                outf.println(gson.toJson(obj));
+            });
+            JsonObject obj = new JsonObject();
+            // exactly what is in CLDR .txt file
+            obj.add("coverageLevels", gson.toJsonTree(covlocs));
+
+            // resolved, including all available locales
+            obj.add("effectiveCoverageLevels", gson.toJsonTree(effectiveCovlocs));
+            outf.println(gson.toJson(obj));
         }
     }
 
@@ -1213,15 +1220,23 @@ public void writePackageList(String outputDir) throws IOException {
             } else {
                 {
                     JsonObject packageEntry = new JsonObject();
-                    packageEntry.addProperty("description", e.getValue() + " (full)");
+                    packageEntry.addProperty("description", e.getValue() + " (basic)");
+                    packageEntry.addProperty("tier", "full");
+                    packageEntry.addProperty("name", CLDR_PKG_PREFIX + baseName + FULL_TIER_SUFFIX);
+                    packages.add(packageEntry);
+                    pkgsToDesc.put(packageEntry.get("name").getAsString(), packageEntry.get("description").getAsString());
+                }
+                {
+                    JsonObject packageEntry = new JsonObject();
+                    packageEntry.addProperty("description", e.getValue() + " (basic)");
                     packageEntry.addProperty("tier", "full");
                     packageEntry.addProperty("name", CLDR_PKG_PREFIX + baseName + FULL_TIER_SUFFIX);
                     packages.add(packageEntry);
                     pkgsToDesc.put(packageEntry.get("name").getAsString(), packageEntry.get("description").getAsString());
                 }
                 {
                     JsonObject packageEntry = new JsonObject();
-                    packageEntry.addProperty("description", e.getValue() + " (modern only)");
+                    packageEntry.addProperty("description", e.getValue() + " (modern)");
                     packageEntry.addProperty("tier", "modern");
                     packageEntry.addProperty("name", CLDR_PKG_PREFIX + baseName + MODERN_TIER_SUFFIX);
                     packages.add(packageEntry);
@@ -1716,7 +1731,7 @@ public void processDirectory(String dirName, DraftStatus minimalDraftStatus)
                 }
                 return new Pair<>(dirName + "/" + filename, totalForThisFile);
             })
-            .filter(p -> p.getSecond() == 0)
+            .filter(p -> p.getSecond() == 0) // filter out only files which produced no output
             .map(p -> p.getFirst())
             .toArray();
         System.out.println(progressPrefix(total, total) + " Completed parallel process of " + total + " file(s)");

tools/cldr-code/src/main/java/org/unicode/cldr/json/LdmlConvertRules.java

@@ -313,6 +313,8 @@ public static class SplittableAttributeSpec {
         new SplittableAttributeSpec("measurementSystem-category-temperature", "territories", "type"),
         new SplittableAttributeSpec("paperSize", "territories", "type"),
         new SplittableAttributeSpec("parentLocale", "locales", "parent"),
+        new SplittableAttributeSpec("collations", "locales", "parent"), // parentLocale component=collations
+        new SplittableAttributeSpec("segmentations", "locales", "parent"), // parentLocale component=segmentations
         new SplittableAttributeSpec("hours", "regions", null),
         new SplittableAttributeSpec("dayPeriodRules", "locales", null),
         // new SplittableAttributeSpec("group", "contains", "group"),

tools/cldr-code/src/main/java/org/unicode/cldr/util/CalculatedCoverageLevels.java

@@ -13,6 +13,11 @@
 
 
 public class CalculatedCoverageLevels {
+    /**
+     * Assumed level for root. CLDR-16420
+     */
+    final private static Level DEFAULT_ROOT_LEVEL = Level.MODERN;
+
     final Map<String, Level> levels;
 
     protected CalculatedCoverageLevels(Map<String, Level> levels) {
@@ -26,6 +31,30 @@ public Map<String, Level> getLevels() {
         return levels;
     }
 
+    public Level getEffectiveCoverageLevel(String locale) {
+        return getEffectiveCoverageLevel(CLDRLocale.getInstance(locale));
+    }
+
+    public Level getEffectiveCoverageLevel(CLDRLocale locale) {
+        // per spec, assumed level for the explicit root locale
+        if (locale == CLDRLocale.ROOT) {
+            return DEFAULT_ROOT_LEVEL;
+        }
+        // See if there is an explicit entry
+        final Level level = levels.get(locale.getBaseName());
+        if (level != null) {
+            return level;
+        }
+        // Otherwise, tail-recurse on parent, unless the parent is root
+        final CLDRLocale parent = locale.getParent();
+        if (parent == CLDRLocale.ROOT) {
+            // not found: no level.
+            // TODO: should this really be 'core'? if at least core? CLDR-16420
+            return null;
+        }
+        return getEffectiveCoverageLevel(parent);
+    }
+
     /**
      * Is the locale present in the list? If so, it is at least basic.
      * Requires an exact match on the locale

tools/cldr-code/src/main/java/org/unicode/cldr/util/Level.java

@@ -160,4 +160,14 @@ public static Level max(Level... levels) {
         }
         return result;
     }
+
+    /**
+     * Returns true if this is at least the other level.
+     * Example:  lev.isAtLeast(Level.MODERN)
+     * @param other
+     * @return
+     */
+    public boolean isAtLeast(Level other) {
+        return getLevel() >= other.getLevel();
+    }
 }

tools/cldr-code/src/main/resources/org/unicode/cldr/json/JSON_config_supplemental.txt

@@ -26,7 +26,7 @@ section=weekData ; path=//cldr/supplemental/weekData/.* ; package=core
 section=timeData ; path=//cldr/supplemental/timeData/.* ; package=core
 section=measurementData ; path=//cldr/supplemental/measurementData/.* ; package=core
 section=codeMappings ; path=//cldr/supplemental/codeMappings/.* ; package=core
-section=parentLocales ; path=//cldr/supplemental/parentLocales/.* ; package=core
+section=parentLocales ; path=//cldr/supplemental/parentLocales.* ; package=core
 section=references ; path=//cldr/supplemental/references/.* ; package=core
 section=telephoneCodeData ; path=//cldr/supplemental/telephoneCodeData/.* ; package=core
 section=windowsZones ; path=//cldr/supplemental/windowsZones/.* ; package=core

tools/cldr-code/src/main/resources/org/unicode/cldr/json/pathTransforms.txt

@@ -170,3 +170,6 @@
 < (.*/personNames)/(sampleName)\[@item="([^"]*)"\]/nameField\[@type="([^"]*)"\]$
 > $1/$2/$3/$4
 
+# ParentLocales
+< (.*/parentLocales)\[@component="([^"]*)"\]/(parentLocale)(.*)$
+> $1/$2$4

tools/cldr-code/src/test/java/org/unicode/cldr/util/TestCalculatedCoverageLevels.java

@@ -2,10 +2,22 @@
 
 import static org.junit.jupiter.api.Assertions.assertAll;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.params.provider.Arguments.arguments;
+
 
 public class TestCalculatedCoverageLevels {
     @Test
@@ -21,4 +33,45 @@ void TestBasic() {
         () -> assertTrue(CalculatedCoverageLevels.getInstance().isLocaleAtLeastBasic("fr"), "fr=true"),
         () -> assertFalse(CalculatedCoverageLevels.getInstance().isLocaleAtLeastBasic("und"), "und=false"));
     }
+
+    public static Stream<Arguments> allLocales() {
+        return CLDRConfig.getInstance().getFullCldrFactory().getAvailable().stream().map(str -> arguments(str));
+    }
+
+    @ParameterizedTest(name = "{0}")
+    @MethodSource("allLocales")
+    @Test
+    @Disabled
+    /**
+     * Test compares coverage level to the Locales.txt goals.
+     * Skipped because it fails (for cv, as of v43), but there may be a useful form for it in the future
+     * @param locale
+     */
+    void TestCoverageLevelsVsCldrOrg(final String locale) {
+        final Level coverageLevel = CalculatedCoverageLevels.getInstance().getLevels().get(locale);
+        assumeTrue(coverageLevel != null, () -> "CalculatedCoverageLevel not available for " + locale);
+        final Level localeCoverageLevel = StandardCodes.make().getHighestLocaleCoverageLevel("Cldr", locale);
+        assumeTrue(localeCoverageLevel != null, () -> "Cldr membership not available for " + locale);
+        assertTrue(localeCoverageLevel.compareTo(coverageLevel) >= 0,
+            () -> String.format("Expected Locales.txt:Cldr=%s at least coverageLevels=%s", coverageLevel.name(), localeCoverageLevel.name()));
+    }
+
+    @ParameterizedTest
+    @CsvSource({
+        "en_US, modern",
+        "en, modern",
+        "mul, null",
+        "root, modern",
+        "sr_Cyrl, modern",
+        "sr_Cyrl_XK, modern",
+        "sr, modern",
+    })
+    void TestEffective(final String loc, String l) {
+        Level level = null;
+        if (!l.equals("null")) {
+            level = Level.fromString(l.toUpperCase());
+        }
+        CalculatedCoverageLevels ccl = CalculatedCoverageLevels.getInstance(); // TODO: data sensitivity.  Ideally would use a private instance of CCL from a static file.
+        assertEquals(level, ccl.getEffectiveCoverageLevel(loc));
+    }
 }