Parse the old dumps (#1091)

* Do not fall back when looking for 4.1 data or later

* spots

* Parse all versions, not just the ones that correspond to an age

* no toList

* no 2.1.0

* Another one

* Throw

* meow

* meow

* Nasty bug

* Try not to run out of memory.

* Is this the accursed 13.1 again?

* stray return

* blarg

* Works better than eggspected

* noncharacter

* bring back the cleanup

* spots

* bad regex

* bad limit

Author: eggrobin

unicodetools/src/main/java/org/unicode/props/PropertyParsingInfo.java

@@ -27,6 +27,7 @@
 import org.unicode.props.PropertyUtilities.Merge;
 import org.unicode.props.UcdLineParser.IntRange;
 import org.unicode.props.UcdLineParser.UcdLine.Contents;
+import org.unicode.props.UcdPropertyValues.Binary;
 import org.unicode.text.utility.Settings;
 import org.unicode.text.utility.Utility;
 
@@ -105,7 +106,7 @@ public PropertyParsingInfo(
         this.special = special;
     }
 
-    static final Pattern VERSION = Pattern.compile("v\\d+\\.\\d+");
+    static final Pattern VERSION = Pattern.compile("v\\d+(\\.\\d+)+");
 
     private static void fromStrings(String... propertyInfo) {
         if (propertyInfo.length < 2 || propertyInfo.length > 4) {
@@ -463,7 +464,7 @@ enum FileType {
         NamedSequences,
         NameAliases,
         StandardizedVariants,
-        Confusables
+        Confusables,
     }
 
     static Map<String, FileType> file2Type = new HashMap<String, FileType>();
@@ -561,11 +562,18 @@ static void parseSourceFile(
                             propInfoSet);
                     break;
                 case PropertyValue:
-                    parsePropertyValueFile(
-                            parser.withMissing(true),
-                            fileName,
-                            indexUnicodeProperties,
-                            nextProperties);
+                    if (fileName.equals("PropList")
+                            && indexUnicodeProperties.ucdVersion.compareTo(
+                                            VersionInfo.UNICODE_3_1_0)
+                                    < 0) {
+                        parsePropertyDumpFile(fullFilename, indexUnicodeProperties, nextProperties);
+                    } else {
+                        parsePropertyValueFile(
+                                parser.withMissing(true),
+                                fileName,
+                                indexUnicodeProperties,
+                                nextProperties);
+                    }
                     break;
                 case Confusables:
                     parseConfusablesFile(
@@ -769,6 +777,45 @@ private static void parseNamedSequencesFile(
         }
     }
 
+    private static void parsePropertyDumpFile(
+            String fullFilename,
+            IndexUnicodeProperties indexUnicodeProperties,
+            IndexUnicodeProperties nextProperties) {
+        final var dumpHeading = Pattern.compile("Property dump for: 0x[0-9A-F]{8} \\(([^()]+)\\)");
+        final var dataLine =
+                Pattern.compile("[0-9A-F]{4,6}(\\.\\.[0-9A-F]{4,6} +\\(\\d+ chars\\))?");
+        PropertyParsingInfo propInfo = null;
+        for (String line : FileUtilities.in("", fullFilename)) {
+            final var heading = dumpHeading.matcher(line);
+            if (heading.matches()) {
+                String name = heading.group(1);
+                propInfo = property2PropertyInfo.get(UcdProperty.forString(name));
+                if (propInfo == null) {
+                    if (name.equals("Not a Character")) {
+                        // Appears in 3.0.1.  See also 84-M6 and 84-M7.
+                        propInfo = property2PropertyInfo.get(UcdProperty.Noncharacter_Code_Point);
+                    } else {
+                        System.err.println("Ignoring unknown property in dump: " + name);
+                    }
+                }
+                continue;
+            }
+            if (propInfo != null && dataLine.matcher(line).matches()) {
+                var range = new UcdLineParser.IntRange();
+                range.set(line.split(" ", 2)[0]);
+                final var data = indexUnicodeProperties.property2UnicodeMap.get(propInfo.property);
+                propInfo.put(
+                        data,
+                        range,
+                        "Yes",
+                        null,
+                        nextProperties == null
+                                ? null
+                                : nextProperties.getProperty(propInfo.property));
+            }
+        }
+    }
+
     private static void parsePropertyValueFile(
             UcdLineParser parser,
             String filename,

unicodetools/src/main/java/org/unicode/text/utility/Utility.java

@@ -1546,11 +1546,6 @@ public static String getMostRecentUnicodeDataFile(
                             + fileType
                             + "'");
         }
-        if ((versionString.startsWith("2.") || versionString.startsWith("3.0"))
-                && filename.startsWith("Prop")) {
-            // Ignore the property dumps for now, as we do not have parsing logic for them.
-            return null;
-        }
         return result;
     }
 

unicodetools/src/main/resources/org/unicode/props/IndexUnicodeProperties.txt

@@ -129,7 +129,9 @@ SpecialCasing ; Lowercase_Mapping ; 1 ; SkipAny4
 SpecialCasing ; Titlecase_Mapping ; 2 ; SkipAny4
 SpecialCasing ; Uppercase_Mapping ; 3 ; SkipAny4
 DerivedCoreProperties ; Lowercase
+PropList ; Lowercase ; v3.0.1
 DerivedCoreProperties ; Uppercase
+PropList ; Uppercase ; v3.0.1
 DerivedCoreProperties ; Cased
 DerivedCoreProperties ; Case_Ignorable
 DerivedCoreProperties ; Changes_When_Lowercased
@@ -138,13 +140,15 @@ DerivedCoreProperties ; Changes_When_Titlecased
 DerivedCoreProperties ; Changes_When_Casefolded
 DerivedCoreProperties ; Changes_When_Casemapped
 DerivedCoreProperties ; Alphabetic
+PropList ; Alphabetic ; v3.0.1
 DerivedCoreProperties ; Default_Ignorable_Code_Point
 DerivedCoreProperties ; Grapheme_Base
 DerivedCoreProperties ; Grapheme_Extend
 # Deprecated and made derived in 5.0.
 DerivedCoreProperties ; Grapheme_Link
 PropList ; Grapheme_Link ; v4.1
 DerivedCoreProperties ; Math
+PropList ; Math ; v3.0.1
 DerivedCoreProperties ; ID_Start
 DerivedCoreProperties ; ID_Continue
 DerivedCoreProperties ; XID_Start