CLDR-18996 fix GenerateLanguageContainment (#5058)

Author: srl295

docs/site/development/updating-codes/updating-language-groups.md

@@ -2,6 +2,36 @@
 title: Updating Language Groups
 ---
 
-# Updating Language Groups
-
-This file has not yet been converted: see https://sites.google.com/unicode.org/cldr/development/updating-codes/updating-language-groups
+1. (prerequisite: being able to build CLDR locally with [Maven](/development/maven)
+2. Run GenerateLanguageContainment, through eclipse or maven.
+   Here is how you can run it with Maven:
+   1. cd cldr/tools
+   2. mvn \-DCLDR\_DIR=*/path/to/***cldr**  \-Dexec.mainClass=org.unicode.cldr.tool.**GenerateLanguageContainment** exec:java \-pl cldr-rdf
+3. This will create {workspace}/cldr/common/supplemental/languageGroup.xml
+   1. Copy the console log into debugLog.txt to help in debugging problems. (Should modify tool to do this.)
+   2. Run TestLanguageGroup and fix problems if necessary:
+   3. OVERRIDES: If a language code moves or is deleted, consider adding override to GenerateLanguageContainment
+      1. Additions go in EXTRA\_PARENT\_CHILDREN
+         1. If you add something, you might have to remove it someplace else. You'll get a "duplicate parent" error in TestLanguageGroup
+         2. Removals go in REMOVE\_PARENT\_CHILDREN
+            1. "\*" for value means all.
+   4. Example: pcm \[Nigerian Pidgin\] \[pcm\] \- not in languages/isolates.json nor languageGroup.xml
+      1. Go to [https://en.wikipedia.org/wiki/Nigerian\_Pidgin](https://en.wikipedia.org/wiki/Nigerian_Pidgin) (by searching)
+         2. Under language family, click on the ancestor. Keep clicking until you find a language group with an "[**ISO 639-2**](https://en.wikipedia.org/wiki/ISO_639-2) **/ [5](https://en.wikipedia.org/wiki/ISO_639-5)**" code.
+         3. Get the ancestor chain (see below), we find kri
+         4. Go to GenerateLanguageContainment.EXTRA\_PARENT\_CHILDREN, add .put("kri", "pcm")
+   5. Example: inc \[Indic\] is not an ancestor of trw \[Torwali\]: expected true
+      1. Go to [https://en.wikipedia.org/wiki/Torwali\_language](https://en.wikipedia.org/wiki/Torwali_language) (find by searching).
+         1. Under language family, click on the ancestor. Keep clicking until you find a language group with an "[**ISO 639-2**](https://en.wikipedia.org/wiki/ISO_639-2) **/ [5](https://en.wikipedia.org/wiki/ISO_639-5)**" code.
+         2. That says 'inc', so we have a case where wikidata is out of sync with wikipedia.
+         3. Go to GenerateLanguageContainment.EXTRA\_PARENT\_CHILDREN, add .put("inc", "trw")
+   6. Occasionally LanguageGroup.java will need some fixes instead, once you have done the research.
+      1. Once you are done, rerun GenerateLanguageContainment and TestLanguageGroup
+         1. You may need to repeat the process to get a full chain of ancestors.
+         2. Example: For X Creoles, we use the X, so for the first example above we needed .put("en", "kri")
+4. Run the tool **ChartLanguageGroups**
+   1. Review {workspace}/../cldr-staging/docs/charts/*\<release\>*/supplemental/language\_groups.html
+   2. Check in
+      1. {workspace}/cldr/common/supplemental/languageGroup.xml
+      2. {workspace}/cldr/tools/cldr-rdf/external/\*.tsv *( intermediate tables, for tracking)*
+      3. Chart: {workspace}/../cldr-staging/docs/charts/*\<release\>*/supplemental/language\_groups.html

tools/cldr-code/src/main/java/org/unicode/cldr/util/CLDRFile.java

@@ -2917,6 +2917,9 @@ public String getWinningPath(String path) {
      * @return
      */
     public Collection<String> getExtraPaths() {
+        if (this.getDtdType() != DtdType.ldml) {
+            return Collections.emptySet();
+        }
         Set<String> toAddTo = new HashSet<>(getRawExtraPaths());
         for (String path : this.iterableWithoutExtras()) {
             toAddTo.remove(path);
@@ -2930,6 +2933,9 @@ public Collection<String> getExtraPaths() {
      * @return
      */
     public Collection<String> getExtraPaths(String prefix, Collection<String> toAddTo) {
+        if (this.getDtdType() != DtdType.ldml) {
+            return Collections.emptySet();
+        }
         for (String item : getRawExtraPaths()) {
             if (item.startsWith(prefix)
                     && dataSource.getValueAtPath(item) == null) { // don't use getStringValue, since
@@ -2949,6 +2955,9 @@ public Collection<String> getExtraPaths(String prefix, Collection<String> toAddT
      * @return
      */
     public Set<String> getRawExtraPaths() {
+        if (this.getDtdType() != DtdType.ldml) {
+            return Collections.emptySet();
+        }
         if (extraPaths == null) {
             extraPaths = ImmutableSet.<String>builder().addAll(getRawExtraPathsPrivate()).build();
             if (DEBUG) {

tools/cldr-code/src/main/java/org/unicode/cldr/util/DiffLanguageGroups.java

@@ -280,7 +280,9 @@ public static void showDiff(String title, Set<String> oldMinusOther) {
     }
 
     public static String show(String languageCode) {
-        return languageCode.equals("mul") ? "Ω" : getName(languageCode) + " ⁅" + languageCode + "⁆";
+        return languageCode.equals("mul")
+                ? languageCode
+                : getName(languageCode) + " ⁅" + languageCode + "⁆";
     }
 
     public static String getName(String languageCode) {

tools/cldr-code/src/main/java/org/unicode/cldr/util/DtdData.java

@@ -1023,8 +1023,16 @@ public int xpathComparator(XPathParts a, XPathParts b) {
             String baseA = a.getElement(0);
             String baseB = b.getElement(0);
             if (!ROOT.name.equals(baseA) || !ROOT.name.equals(baseB)) {
+                final XPathParts oddity = (ROOT.name.equals(baseA) ? b : a);
+                final String baseOddity = oddity.getElement(0);
+                final String elementOddity = oddity.getElement(-1);
                 throw new IllegalArgumentException(
-                        "Comparing different DTDs: " + ROOT.name + ", " + baseA + ", " + baseB);
+                        "Comparing different DTDs: This comparator is for DTD "
+                                + ROOT.name
+                                + ", but attempted compare with DTD "
+                                + baseOddity
+                                + " in: "
+                                + oddity.toString());
             }
             int min = Math.min(a.size(), b.size());
             Element parent = ROOT;

tools/cldr-code/src/main/java/org/unicode/cldr/util/TempPrintWriter.java

@@ -19,6 +19,9 @@
  * the old file (except for date), then it is deleted. Otherwise it replaces the target file. Moved
  * from UnicodeTools.
  *
+ * <p>dontReplaceFile() may be called before close() to prevent replacement (such as if a processing
+ * error occurred)
+ *
  * @author markdavis
  */
 public class TempPrintWriter extends Writer {
@@ -55,6 +58,10 @@ public static TempPrintWriter openUTF8Writer(String dir, String filename) {
         return new TempPrintWriter(new File(dir, filename));
     }
 
+    public static TempPrintWriter openUTF8Writer(File dir, String filename) {
+        return new TempPrintWriter(new File(dir, filename));
+    }
+
     public TempPrintWriter(String dir, String filename) {
         this(new File(dir, filename));
     }
@@ -76,6 +83,10 @@ public TempPrintWriter(File file) {
         }
     }
 
+    /**
+     * Will prevent the file from being overwritten. Call this before close() if something goes
+     * wrong during write.
+     */
     public void dontReplaceFile() {
         noReplace = true;
     }

tools/cldr-rdf/src/main/java/org/unicode/cldr/tool/GenerateLanguageContainment.java

@@ -50,6 +50,7 @@
 import org.unicode.cldr.util.StandardCodes;
 import org.unicode.cldr.util.StandardCodes.LstrField;
 import org.unicode.cldr.util.StandardCodes.LstrType;
+import org.unicode.cldr.util.TempPrintWriter;
 import org.unicode.cldr.util.Validity;
 import org.unicode.cldr.util.Validity.Status;
 
@@ -461,8 +462,8 @@ void run(String[] args) throws IOException {
             }
         }
         System.out.println("Writing " + "skippingCodes.tsv");
-        try (PrintWriter w =
-                FileUtilities.openUTF8Writer(TsvWriter.getTsvDir(), "skippingCodes.tsv")) {
+        try (TempPrintWriter w =
+                TempPrintWriter.openUTF8Writer(TsvWriter.getTsvDir(), "skippingCodes.tsv")) {
             // TsvWriter.writeRow(w, "childCode\tLabel", "parentCode\tLabel"); // header
             skipping.forEach(e -> w.println(e));
         }
@@ -568,12 +569,15 @@ void run(String[] args) throws IOException {
         newFile.add("//" + DtdType.supplementalData + "/version[@number='$Revision$']", "");
         printXML(newFile, parentToChild);
 
-        try (PrintWriter outFile =
-                FileUtilities.openUTF8Writer(
+        try (TempPrintWriter outFile =
+                TempPrintWriter.openUTF8Writer(
                         CLDRPaths.SUPPLEMENTAL_DIRECTORY, "languageGroup.xml")) {
-            newFile.write(outFile);
-        } catch (IOException e1) {
-            throw new ICUUncheckedIOException("Can't write to languageGroup.xml", e1);
+            try {
+                newFile.write(outFile.asPrintWriter());
+            } catch (Throwable e1) {
+                outFile.dontReplaceFile();
+                throw new ICUUncheckedIOException("Can't write to languageGroup.xml", e1);
+            }
         }
 
         // for (Entry<String,String> entry : childToParent.entries()) {