Actually check values of enumerated properties (#1083)

Co-authored-by: Markus Scherer <[email protected]>

Author: eggrobin

unicodetools/src/main/java/org/unicode/props/IndexUnicodeProperties.java

@@ -925,11 +925,7 @@ public static void loadUcdHistory(
                 final var properties = IndexUnicodeProperties.make(age.getShortName());
                 for (UcdProperty property : UcdProperty.values()) {
                     if (property.getShortName().startsWith("cjk") == unihan) {
-                        try {
-                            properties.load(property, expectCacheHit);
-                        } catch (ICUException e) {
-                            e.printStackTrace();
-                        }
+                        properties.load(property, expectCacheHit);
                     }
                 }
                 System.out.println(

unicodetools/src/main/java/org/unicode/props/PropertyParsingInfo.java

@@ -256,7 +256,13 @@ public void put(
             Merge<String> merger,
             boolean hackHangul,
             UnicodeProperty nextVersion) {
-        // MEOW
+        if (value == null && property == UcdProperty.Idn_2008) {
+            // The IDNA2008 Status field of the IDNA mapping table is treated as an enumerated
+            // property by the tools, with an Extra @missing line with a value na.
+            // Unusually, the file has data lines with no IDNA2008 Status field; the default should
+            // apply to these ranges.
+            return;
+        }
         if (value != null
                 && value.isEmpty()
                 && property != UcdProperty.NFKC_Casefold
@@ -409,8 +415,8 @@ public String checkRegex2(String string) {
     public String checkEnum(String string) {
         final Enum item = string == null ? null : property.getEnum(string);
         if (item == null) {
-            final String errorMessage = property + "\tBad enum value:\t" + string;
-            IndexUnicodeProperties.getDataLoadingErrors().put(property, errorMessage);
+            throw new UnicodePropertyException(
+                    "\tBad enum value for " + property + " :\t" + string);
         } else {
             string = item.toString();
         }
@@ -746,9 +752,37 @@ private static void parsePropertyValueFile(
             IndexUnicodeProperties indexUnicodeProperties,
             IndexUnicodeProperties nextProperties) {
         for (UcdLineParser.UcdLine line : parser) {
+            if (line.getOriginalLine().equals("U+")
+                    && filename.startsWith("Unihan")
+                    && indexUnicodeProperties.ucdVersion.getMajor() == 2) {
+                // Truncated Unihan-1.txt in Unicode 2.0.
+                return;
+            }
             String propName = line.getParts()[1];
             UcdProperty item = UcdProperty.forString(propName);
 
+            String extractedValue = null;
+            if (item == null
+                    && indexUnicodeProperties.ucdVersion.compareTo(VersionInfo.UNICODE_4_0) <= 0) {
+                // DerivedNormalizationProps Version 4.0 and earlier is highly irregular.
+                // It provides the NFMeow_QC assignments as a single field NFMEOW_Value,
+                // and calls FC_NFKC_Closure FNC.  Since we need special handling for the former,
+                // we also deal with FNC here instead of making it an Extra alias.
+                if (propName.equals("FNC")) {
+                    propName = "FC_NFKC_Closure";
+                    item = UcdProperty.forString(propName);
+                } else {
+                    String[] parts = propName.split("_");
+                    if (parts.length == 2
+                            && (parts[1].equals("NO") | parts[1].equals("MAYBE"))
+                            && (parts[0].startsWith("NF"))) {
+                        propName = parts[0] + "_QC";
+                        item = UcdProperty.forString(propName);
+                        extractedValue = parts[1];
+                    }
+                }
+            }
+
             if (item == null) {
                 throw new IllegalArgumentException(
                         "Missing property enum in UcdProperty for "
@@ -773,9 +807,10 @@ private static void parsePropertyValueFile(
             //                    }
             String value;
             // The file emoji-sequences.txt has a comment-like field after the binary property.
-            if (line.getParts().length == 2
-                    || filename.equals("emoji/*/emoji-sequences")
-                    || filename.equals("emoji/*/emoji-zwj-sequences")) {
+            if (extractedValue == null
+                    && (line.getParts().length == 2
+                            || filename.equals("emoji/*/emoji-sequences")
+                            || filename.equals("emoji/*/emoji-zwj-sequences"))) {
                 if (propInfo.property.getType() != PropertyType.Binary) {
                     throw new IllegalArgumentException(
                             "Expected a value for "
@@ -785,7 +820,14 @@ private static void parsePropertyValueFile(
                 }
                 value = "Yes";
             } else {
-                value = line.getParts()[2];
+                value = extractedValue != null ? extractedValue : line.getParts()[2];
+                if (propInfo.property == UcdProperty.kJapaneseOn
+                        && indexUnicodeProperties.ucdVersion.equals(VersionInfo.UNICODE_3_1_0)
+                        && value.isEmpty()
+                        && line.getParts().length == 4) {
+                    // Extra tab in the kJapaneseOn record for U+4E00.
+                    value = line.getParts()[3];
+                }
                 if (propInfo.property.getType() == PropertyType.Binary) {
                     if (line.getType() == Contents.DATA
                             && UcdPropertyValues.Binary.forName(value)
@@ -817,7 +859,10 @@ private static void parsePropertyValueFile(
                         && !(propInfo.property == UcdProperty.NFKC_Casefold
                                 || propInfo.property == UcdProperty.NFKC_Simple_Casefold)) {
                     throw new IllegalArgumentException(
-                            "Unexpected empty value for property " + propName);
+                            "Unexpected empty value for property "
+                                    + propName
+                                    + ":  "
+                                    + line.getOriginalLine());
                 }
                 if (propInfo.property == UcdProperty.kMandarin) {
                     if (indexUnicodeProperties.oldVersion) {
@@ -941,26 +986,6 @@ private static void parseConfusablesFile(
                     intRange,
                     parts[1],
                     nextProperties == null ? null : nextProperties.getProperty(propInfo.property));
-            intRange.set(parts[1]);
-            if (intRange.string == null) {
-                if (!data.containsKey(intRange.start)) {
-                    propInfo.put(
-                            data,
-                            intRange,
-                            parts[1],
-                            nextProperties == null
-                                    ? null
-                                    : nextProperties.getProperty(propInfo.property));
-                }
-            } else if (!intRange.string.isEmpty() && !data.containsKey(intRange.string)) {
-                propInfo.put(
-                        data,
-                        intRange,
-                        parts[1],
-                        nextProperties == null
-                                ? null
-                                : nextProperties.getProperty(propInfo.property));
-            }
         }
     }
 
@@ -1111,7 +1136,12 @@ private static void parseFields(
                         throw new UnicodePropertyException();
                 }
                 String value =
-                        propInfo.fieldNumber >= parts.length ? "" : parts[propInfo.fieldNumber];
+                        propInfo.fieldNumber >= parts.length ? null : parts[propInfo.fieldNumber];
+                if (propInfo.property == UcdProperty.Joining_Group
+                        && indexUnicodeProperties.ucdVersion.compareTo(VersionInfo.UNICODE_4_0) <= 0
+                        && value.equals("<no shaping>")) {
+                    value = "No_Joining_Group";
+                }
                 propInfo.put(
                         data,
                         line.getMissingSet(),
@@ -1172,16 +1202,32 @@ private static void parseSimpleFieldFile(
                     IntRange range = new IntRange();
                     range.start = Utility.codePointFromHex(line.getParts()[0]);
                     range.end = Utility.codePointFromHex(line.getParts()[1]);
+                    // Unicode 2 puts FEFF both in Arabic Presentation Forms-B and in Specials.
+                    // We are not going to make Block multivalued for that, so we let the second
+                    // assignment win.
+                    // This fits with assignments in Unicode 2.1.4..3.1.1 where
+                    // Arabic Presentation Forms-B ended on FEFE and Specials was a
+                    // split Block of FEFF & FFF0..FFFD.
+                    // Since Unicode 3.2, blocks were contiguous xxx0..yyyF:
+                    // https://www.unicode.org/reports/tr28/tr28-3.html#database
+                    // The normative blocks defined in Blocks.txt have been adjusted slightly,
+                    // in accordance with Unicode Technical Committee decisions.
+                    // - Every block starts and ends on a column boundary.
+                    //   That is, the last digit of the first code point in the block is always 0,
+                    //   and the last digit of the final code point in the block is always F.
+                    // - Every block is contiguous. [...]
                     propInfo.put(
                             data,
                             line.getMissingSet(),
                             range,
                             line.getParts()[2],
-                            null,
+                            version.getMajor() == 2 ? new PropertyUtilities.Overrider() : null,
                             false,
                             nextVersion);
                     continue;
-                } else if (line.getParts().length != 2) {
+                } else if (line.getParts().length != 2
+                        && version.compareTo(VersionInfo.UNICODE_3_0) > 0) {
+                    // Unicode 3.0 and earlier had name comments as an extra field.
                     throw new IllegalArgumentException(
                             "Too many fields in " + line.getOriginalLine());
                 }

unicodetools/src/main/java/org/unicode/props/PropertyUtilities.java

@@ -28,6 +28,15 @@ public String merge(String first, String second) {
         }
     }
 
+    public static final class Overrider implements Merge<String> {
+        public Overrider() {}
+
+        @Override
+        public String merge(String first, String second) {
+            return second;
+        }
+    }
+
     static final <K, V, M extends Map<K, V>> M putNew(M map, K key, V value) {
         final V oldValue = map.get(key);
         if (oldValue != null) {

unicodetools/src/main/java/org/unicode/props/UcdProperty.java

@@ -362,7 +362,12 @@ public enum UcdProperty {
     Quotation_Mark(PropertyType.Binary, Binary.class, null, "QMark"),
     RGI_Emoji_Flag_Sequence(PropertyType.Binary, Binary.class, null, "REFS", "Emoji_Flag_Sequence"),
     RGI_Emoji_Keycap_Sequence(
-            PropertyType.Binary, Binary.class, null, "REKS", "Emoji_Keycap_Sequence"),
+            PropertyType.Binary,
+            Binary.class,
+            null,
+            "REKS",
+            "Emoji_Keycap_Sequence",
+            "Emoji_Combining_Sequence"),
     RGI_Emoji_Modifier_Sequence(
             PropertyType.Binary, Binary.class, null, "REMS", "Emoji_Modifier_Sequence"),
     RGI_Emoji_Tag_Sequence(PropertyType.Binary, Binary.class, null, "RETS", "Emoji_Tag_Sequence"),

unicodetools/src/main/java/org/unicode/props/UcdPropertyValues.java

@@ -610,7 +610,105 @@ public enum Canonical_Combining_Class_Values implements Named {
         Above_Right("232", "AR"),
         Double_Below("233", "DB"),
         Double_Above("234", "DA"),
-        Iota_Subscript("240", "IS");
+        Iota_Subscript("240", "IS"),
+        CCC37("37"),
+        CCC38("38"),
+        CCC39("39"),
+        CCC40("40"),
+        CCC41("41"),
+        CCC42("42"),
+        CCC43("43"),
+        CCC44("44"),
+        CCC45("45"),
+        CCC46("46"),
+        CCC47("47"),
+        CCC48("48"),
+        CCC49("49"),
+        CCC50("50"),
+        CCC51("51"),
+        CCC52("52"),
+        CCC53("53"),
+        CCC54("54"),
+        CCC55("55"),
+        CCC56("56"),
+        CCC57("57"),
+        CCC58("58"),
+        CCC59("59"),
+        CCC60("60"),
+        CCC61("61"),
+        CCC62("62"),
+        CCC63("63"),
+        CCC64("64"),
+        CCC65("65"),
+        CCC66("66"),
+        CCC67("67"),
+        CCC68("68"),
+        CCC69("69"),
+        CCC70("70"),
+        CCC71("71"),
+        CCC72("72"),
+        CCC73("73"),
+        CCC74("74"),
+        CCC75("75"),
+        CCC76("76"),
+        CCC77("77"),
+        CCC78("78"),
+        CCC79("79"),
+        CCC80("80"),
+        CCC81("81"),
+        CCC82("82"),
+        CCC83("83"),
+        CCC85("85"),
+        CCC86("86"),
+        CCC87("87"),
+        CCC88("88"),
+        CCC89("89"),
+        CCC90("90"),
+        CCC92("92"),
+        CCC93("93"),
+        CCC94("94"),
+        CCC95("95"),
+        CCC96("96"),
+        CCC97("97"),
+        CCC98("98"),
+        CCC99("99"),
+        CCC100("100"),
+        CCC101("101"),
+        CCC102("102"),
+        CCC104("104"),
+        CCC105("105"),
+        CCC106("106"),
+        CCC108("108"),
+        CCC109("109"),
+        CCC110("110"),
+        CCC111("111"),
+        CCC112("112"),
+        CCC113("113"),
+        CCC114("114"),
+        CCC115("115"),
+        CCC116("116"),
+        CCC117("117"),
+        CCC119("119"),
+        CCC120("120"),
+        CCC121("121"),
+        CCC123("123"),
+        CCC124("124"),
+        CCC125("125"),
+        CCC126("126"),
+        CCC127("127"),
+        CCC128("128"),
+        CCC131("131"),
+        CCC134("134"),
+        CCC135("135"),
+        CCC136("136"),
+        CCC137("137"),
+        CCC138("138"),
+        CCC139("139"),
+        CCC140("140"),
+        CCC141("141"),
+        CCC142("142"),
+        CCC143("143"),
+        CCC144("144");
         private final PropertyNames<Canonical_Combining_Class_Values> names;
 
         private Canonical_Combining_Class_Values(String shortName, String... otherNames) {
@@ -703,7 +801,8 @@ public enum Do_Not_Emit_Type_Values implements Named {
         Precomposed_Form("Precomposed_Form"),
         Deprecated("Deprecated"),
         Discouraged("Discouraged"),
-        Preferred_Spelling("Preferred_Spelling");
+        Preferred_Spelling("Preferred_Spelling"),
+        Arabic_Tashkil("Arabic_Tashkil");
         private final PropertyNames<Do_Not_Emit_Type_Values> names;
 
         private Do_Not_Emit_Type_Values(String shortName, String... otherNames) {
@@ -942,7 +1041,7 @@ public static Identifier_Status_Values forName(String name) {
     }
 
     public enum Identifier_Type_Values implements Named {
-        Not_Character("nc", "not_chars"),
+        Not_Character("nc", "not_chars", "Not_Characters"),
         Deprecated("d"),
         Default_Ignorable("di"),
         Not_NFKC("nn"),
@@ -1093,7 +1192,8 @@ public enum Indic_Positional_Category_Values implements Named {
         Top_And_Left("Top_And_Left"),
         Top_And_Left_And_Right("Top_And_Left_And_Right"),
         Top_And_Right("Top_And_Right"),
-        Visual_Order_Left("Visual_Order_Left");
+        Visual_Order_Left("Visual_Order_Left"),
+        Invisible("Invisible");
         private final PropertyNames<Indic_Positional_Category_Values> names;
 
         private Indic_Positional_Category_Values(String shortName, String... otherNames) {
@@ -1157,7 +1257,8 @@ public enum Indic_Syllabic_Category_Values implements Named {
         Visarga("Visarga"),
         Vowel("Vowel"),
         Vowel_Dependent("Vowel_Dependent"),
-        Vowel_Independent("Vowel_Independent");
+        Vowel_Independent("Vowel_Independent"),
+        Consonant_Repha("Consonant_Repha");
         private final PropertyNames<Indic_Syllabic_Category_Values> names;
 
         private Indic_Syllabic_Category_Values(String shortName, String... otherNames) {
@@ -1370,7 +1471,21 @@ public enum Joining_Group_Values implements Named {
         Yudh("Yudh"),
         Yudh_He("Yudh_He"),
         Zain("Zain"),
-        Zhain("Zhain");
+        Zhain("Zhain"),
+        BAA("BAA"),
+        FA("FA"),
+        HAA("HAA"),
+        HA_GOAL("HA_GOAL"),
+        HA("HA"),
+        CAF("CAF"),
+        KNOTTED_HA("KNOTTED_HA"),
+        RA("RA"),
+        SWASH_CAF("SWASH_CAF"),
+        HAMZAH_ON_HA_GOAL("HAMZAH_ON_HA_GOAL"),
+        TAA_MARBUTAH("TAA_MARBUTAH"),
+        YA_BARREE("YA_BARREE"),
+        YA("YA"),
+        ALEF_MAQSURAH("ALEF_MAQSURAH");
         private final PropertyNames<Joining_Group_Values> names;
 
         private Joining_Group_Values(String shortName, String... otherNames) {