The real 1.1.5 (#1098)

* Do not fall back when looking for 4.1 data or later

* spots

* Parse all versions, not just the ones that correspond to an age

* no toList

* no 2.1.0

* Another one

* Throw

* meow

* meow

* Nasty bug

* Try not to run out of memory.

* Is this the accursed 13.1 again?

* stray return

* blarg

* Works better than eggspected

* noncharacter

* bring back the cleanup

* spots

* bad regex

* bad limit

* Better handling of various kinds of nulls

* spots

* The Real UnicodeData-1.1.5.txt

* Reconstructions

* Fix Age derivation

* The -Update curse

* Read the reconstructed files

* Errors in reconstruction

* Better parsing of UnicodeData versions 1.x and 2.x.

* Another error in reconstruction, cf. https://www.unicode.org/versions/Unicode1.0.0/Notice.pdf

* Do not remove Name

* Remove incorrect and outdated UTS39 data

* Corrected reconstructions from Ken

* Correction to Ken’s correction

Author: eggrobin

UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeUtilities.java

@@ -1385,7 +1385,6 @@ public static void showProperties(
         } else {
             name = toHTML.transliterate(name);
         }
-        boolean allowed = XIDModifications.isAllowed(cp);
 
         String scriptCat = getScriptCat("", cp);
         if (showDevProperties) {
@@ -1415,16 +1414,6 @@ public static void showProperties(
         out.append("<tr><td class='bigCode'>" + hex + "</td></tr>\n");
         out.append("<tr><td class='bigName'>" + name + "</td></tr>\n");
         out.append("<tr><td class='bigName'>" + scriptCat + "</td></tr>\n");
-        out.append("<tr><td class='bigName'><i>id:</i> ");
-        if (allowed) {
-            out.append("<span class='allowed'>allowed</span>");
-        } else {
-            out.append(
-                    "<span class='restricted' title='Restricted in identifiers: "
-                            + XIDModifications.getType(cp)
-                            + "'>restricted</span>");
-        }
-        out.append("</td></tr>\n");
         StringBuilder confusableString = displayConfusables(cp);
         out.append(
                 "<tr><td class='bigName'><span title='Confusable Characters'><i>confuse:</i> </span>");
@@ -1438,7 +1427,7 @@ public static void showProperties(
 
         List<String> availableNames = (List<String>) getFactory().getAvailableNames();
         TreeSet<String> sortedProps =
-                Builder.with(new TreeSet<String>(col)).addAll(availableNames).remove("Name").get();
+                Builder.with(new TreeSet<String>(col)).addAll(availableNames).get();
 
         String kRSUnicode = getFactory().getProperty("kRSUnicode").getValue(cp);
         boolean isUnihan = kRSUnicode != null;
@@ -1484,7 +1473,7 @@ public static void showProperties(
                 history.equals("assigned") && age != Age_Values.Unassigned
                         ? VersionInfo.getInstance(age.getShortName())
                         : history.equals("full")
-                                ? VersionInfo.getInstance(Age_Values.V1_1.getShortName())
+                                ? Utility.UNICODE_VERSIONS.get(Utility.UNICODE_VERSIONS.size() - 1)
                                 : Settings.LAST_VERSION_INFO;
         if (minVersion.compareTo(UcdLoader.getOldestLoadedUcd()) < 0) {
             minVersion = UcdLoader.getOldestLoadedUcd();

unicodetools/data/ucd/1.0.0-Update/UnicodeData.txt

@@ -3,6 +3,9 @@
 import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.text.UnicodeSet;
 
+// TODO(egg): This class seems to be based on an antique version of UTS #39 data, and also to return
+// inverted results (allowed for disallowed and vice-versa).  Consider deleting now that we just
+// show real UTS #39 data in the tools.
 public class XIDModifications {
     private static UnicodeMap<String> allowed = new UnicodeMap(); // "[:XID_Continue:]");
     private static UnicodeMap<String> reasons = new UnicodeMap<String>();

unicodetools/data/ucd/1.0.1-Update/UnicodeData.txt

@@ -1153,6 +1153,51 @@ private static void parseUnicodeDataFile(
                 } else if (parts[1].contains("Hangul Syllable")) {
                     parts[1] = CONSTRUCTED_NAME;
                     hackHangul = true;
+                } else if (parts[1].contains("CJK Compatibility Ideograph")) {
+                    // Unicode 2.0 through 2.1.2 have
+                    // F900;<CJK Compatibility Ideograph, First>;Lo;0;L;;;;;N;;;;;
+                    // FA2D;<CJK Compatibility Ideograph, Last>;Lo;0;L;;;;;N;;;;;
+                    // and this is replicated in the reconstructed 1.0.0 and 1.0.1 files.
+                    parts[1] = "CJK COMPATIBILITY IDEOGRAPH-#";
+                } else if (parts[1].equals("<CJK IDEOGRAPH REPRESENTATIVE>")) {
+                    // UnicodeData-1.1.5.txt does not have ranges yet, instead it has a
+                    // representative that is meant to apply to ranges defined elsewhere.
+                    // We inject these ranges here.
+                    parts[1] = "CJK UNIFIED IDEOGRAPH-#";
+                    // Start is already at 0x4E00, the representative.
+                    line.getRange().end = 0x9FA5;
+                    parseFields(
+                            line,
+                            indexUnicodeProperties,
+                            nextProperties,
+                            propInfoSet,
+                            null,
+                            hackHangul);
+                    line.getRange().start = 0xF900;
+                    line.getRange().end = 0xFA2D;
+                    parts[1] = "CJK COMPATIBILITY IDEOGRAPH-#";
+                    parseFields(
+                            line,
+                            indexUnicodeProperties,
+                            nextProperties,
+                            propInfoSet,
+                            null,
+                            hackHangul);
+                    // UnicodeData-1.1.5.txt is also missing the PUA, which was defined only in
+                    // wording.  Inject it here while we are doing surgery on the surrounding CJK
+                    // blocks.  Note that the PUA has its modern E000..F8FF in 1.1, see
+                    // https://www.unicode.org/versions/Unicode1.1.0/ch02.pdf.
+                    line.getRange().start = 0xE000;
+                    line.getRange().end = 0xF8FF;
+                    parts[1] = null;
+                    parts[2] = "Co";
+                    parseFields(
+                            line,
+                            indexUnicodeProperties,
+                            nextProperties,
+                            propInfoSet,
+                            null,
+                            hackHangul);
                 } else {
                     parts[1] = null;
                 }

unicodetools/data/ucd/1.1.0-Update/UnicodeData.txt

@@ -2241,6 +2241,15 @@ public String getAge(int codePoint) {
                     break;
                 }
                 if (ucdCache[i].isAllocated(codePoint)) {
+                    if (i == UCD_Types.AGE11 && !ucdCache[i + 1].isAllocated(codePoint)) {
+                        // Deallocations in Unicode 2.
+                        continue;
+                    }
+                    return UCD_Names.LONG_AGE[i];
+                } else if (i == UCD_Types.AGE11
+                        && ((codePoint >= 0xE000 && codePoint <= 0xF8FF)
+                                || (codePoint >= 0xF900 && codePoint <= 0xFA2D))) {
+                    // Private use and CJK compatibility ideographs, not overt in UnicodeData 1.1.5.
                     return UCD_Names.LONG_AGE[i];
                 }
             }

unicodetools/src/main/java/org/unicode/jsp/XIDModifications.java

@@ -927,7 +927,9 @@ public static String join(long[] array, String divider) {
         "2.1.5",
         "2.1.2",
         "2.0.0",
-        "1.1.0",
+        "1.1.0", // Really 1.1.5.
+        "1.0.1", // Reconstructed.
+        "1.0.0", // Reconstructed.
     };
 
     public static final List<VersionInfo> UNICODE_VERSIONS =