UCDXML and TR42 v2 (#1030)

* Merged changes manually from ucdxml

* Review changes from Markus

* Merged changes manually from ucdxml

* Review changes from Markus

* Ran spotless

* Ran GenerateEnums

* More review changes from Markus

* Ran spotless

* Use default values where possible

* Corrections from Markus's review

Author: jowilco

.gitignore

@@ -43,6 +43,7 @@ perf-*.xml
 test-*.xml
 
 # Directories
+.idea/
 .settings/
 .vs/
 .vscode/

docs/ucdxml.md

@@ -0,0 +1,69 @@
+# UCDXML
+
+There are three separate processes for generating and validating UCDXML files and their corresponding UAX42 report.
+
+1. Generate the UCDXML files.
+2. (Optional) You can compare the generated UCDXML files against each other (e.g., Flat vs Grouped) or against 
+   previous versions.
+3. Generate UAX42. There are three steps involved:
+
+   1. Generate the property value fragments. The updated versions should live in 
+      unicodetools/src/main/resources/org/unicode/uax42/fragments
+   2. Generate the index.html and index.rnc files for UAX42.
+   3. (Optional) Validate the UCDXML files using index.rnc.
+
+## Generate UCDXML files
+
+- You can generate flat or grouped versions of UCDXML.
+- You can generate UCDXML files for:
+  - the full range of code points
+  - the Unihan code points
+  - code points that are not Unihan code points
+
+```
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range ALL --output FLAT"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range UNIHAN --output FLAT"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range NOUNIHAN --output FLAT"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range ALL --output GROUPED"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range UNIHAN --output GROUPED"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.UCDXML"' '-Dexec.args="--range NOUNIHAN --output GROUPED"' -DCLDR_DIR=$(cd ../cldr; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+```
+
+## Compare UCDXML files
+
+After generating UCDXML files, you can compare:
+
+- Different versions of the same type (range and output) of UCDXML file
+- Grouped and flat versions of the same code point range
+
+```
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.CompareUCDXML"' '-Dexec.args="-a {path to file} -b {path to file}"'
+```
+
+## Generating TR42
+
+### Step 1 - Generate property value fragments
+
+```
+mvn compile exec:java '-Dexec.mainClass="org.unicode.xml.GeneratePropertyValues"' -DCLDR_DIR=$(cd ../cldr ; pwd) -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd) -DUNICODETOOLS_REPO_DIR=$(pwd)
+```
+
+UAX42 fragments live in unicodetools/src/main/resources/org/unicode/uax42/fragments
+
+### Step 2 - Generate TR42 index.html and index.rnc 
+
+```
+mvn xml:transform -f $(cd ./unicodetools/src/main/resources/org/unicode/uax42; pwd) -Doutputdir=$(cd ../Generated/uax42; pwd)
+```
+
+### Step 3 - Validate generated UAX XML files
+
+You'll need a [RELAX NG](https://relaxng.org/) schema validator.
+We'll use [jing-trang](https://github.com/relaxng/jing-trang) in this example.
+
+1. Clone and build [jing-trang](https://github.com/relaxng/jing-trang)
+2. Run the following:
+    ```
+   java -jar C:\_git\jing-trang\build\jing.jar -c UNICODETOOLS_REPO_DIR\uax\uax42\output\index.rnc <path to UAX xml file>
+   ```
+   Note that the UAX xml file has to be saved as NFD as the Unihan syntax regular expressions are expecting NFD.

unicodetools/src/main/java/org/unicode/props/UcdProperty.java

@@ -91,6 +91,7 @@ public enum UcdProperty {
     Named_Sequences_Prov(PropertyType.Miscellaneous, "NSP"),
     Standardized_Variant(PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "SV"),
     Unicode_1_Name(PropertyType.Miscellaneous, "na1"),
+    emoji_variation_sequence(PropertyType.Miscellaneous, "emoji_variation_sequence"),
     kAlternateHanYu(PropertyType.Miscellaneous, "cjkAlternateHanYu"),
     kAlternateJEF(PropertyType.Miscellaneous, "cjkAlternateJEF"),
     kAlternateKangXi(PropertyType.Miscellaneous, "cjkAlternateKangXi"),
@@ -242,6 +243,12 @@ public enum UcdProperty {
     kZhuang(PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "cjkZhuang"),
     kZhuangNumeric(
             PropertyType.Miscellaneous, null, ValueCardinality.Unordered, "cjkZhuangNumeric"),
+    normalization_correction_corrected(
+            PropertyType.Miscellaneous, "normalization_correction_corrected"),
+    normalization_correction_original(
+            PropertyType.Miscellaneous, "normalization_correction_original"),
+    normalization_correction_version(
+            PropertyType.Miscellaneous, "normalization_correction_version"),
 
     // Catalog
     Age(PropertyType.Catalog, Age_Values.class, null, "age"),

unicodetools/src/main/java/org/unicode/props/UcdPropertyValues.java

@@ -766,6 +766,7 @@ public static East_Asian_Width_Values forName(String name) {
     // Emoji_DCM
     // Emoji_KDDI
     // Emoji_SB
+    // emoji_variation_sequence
     // Equivalent_Unified_Ideograph
     // FC_NFKC_Closure
     public enum General_Category_Values implements Named {
@@ -1789,6 +1790,9 @@ public static NFKD_Quick_Check_Values forName(String name) {
         }
     }
 
+    // normalization_correction_corrected
+    // normalization_correction_original
+    // normalization_correction_version
     public enum Numeric_Type_Values implements Named {
         Decimal("De"),
         Digit("Di"),

unicodetools/src/main/java/org/unicode/tools/emoji/LoadImage.java

@@ -891,7 +891,8 @@ public static void doSb(String outputDir) throws IOException {
         //                try {
         //                    copy(new URL(url), new File(outputDir + "/sb","sb_" + code + ".gif"));
         ////                    BufferedImage sourceImage = ImageIO.read(new URL(url));
-        ////                    writeImage(sourceImage,outputDir + "/sb","sb_" + code, "gif");
+        ////                    writeImage(sourceImage,outputDir + "/sb","sb_" + code,
+        // "gif");
         //                    System.out.println(code);
         //                } catch (Exception e) {
         //                    System.out.println("Skipping " + code);