Post-16 invariants (#940)

* Test MCM stability

* Fix line numbers for invariant test failures

* Bindestrich

Co-authored-by: Markus Scherer <markus.icu@gmail.com>

---------

Co-authored-by: Markus Scherer <markus.icu@gmail.com>

Author: eggrobin

unicodetools/src/main/java/org/unicode/text/UCD/TestUnicodeInvariants.java

@@ -184,7 +184,7 @@ public static int testInvariants(String inputFile, String suffix, boolean doRang
                 }
                 final var noComments = new StringBuilder();
                 final List<String> lines = new ArrayList<>();
-                final List<Integer> lineBeginnings = new ArrayList();
+                final List<Integer> lineBeginnings = new ArrayList<>();
                 try (final BufferedReader in = getInputReader(inputFile)) {
                     in.lines()
                             .forEach(
@@ -234,7 +234,24 @@ public static int testInvariants(String inputFile, String suffix, boolean doRang
                         position -> {
                             for (int i = 0; i < lineBeginnings.size(); ++i) {
                                 if (lineBeginnings.get(i) > position.getIndex()) {
-                                    return i; // 1-based line number.
+                                    // The error is before the beginning of line i (0-based), thus
+                                    // on line i (1-based).
+                                    return i;
+                                } else if (lineBeginnings.get(i) == position.getIndex()) {
+                                    // The position in a beginning of line; this happens when a
+                                    // statement has been successfully parsed, but then fails for
+                                    // non-syntactic reasons.
+                                    // The parse position is then the beginning of the next
+                                    // statement.
+                                    // Backtrack to the last nonempty line (ignoring comments),
+                                    // which is the last line of the failing statement.
+                                    int indexInTrimmedSource = position.getIndex();
+                                    while (lineBeginnings.get(i) == indexInTrimmedSource
+                                            && indexInTrimmedSource > 0) {
+                                        --indexInTrimmedSource;
+                                        --i;
+                                    }
+                                    return i + 1;
                                 }
                             }
                             return lineBeginnings.size();

unicodetools/src/main/resources/org/unicode/text/UCD/UnicodeInvariantTest.txt

@@ -746,11 +746,8 @@ In \P{Other_Joining_Type=Deduce_From_General_Category}, Joining_Type = Other_Joi
 [ \p{Other_Joining_Type=Deduce_From_General_Category} & [\p{gc=Mn}\p{gc=Me}\p{gc=Cf}] ] ⊆ \p{Joining_Type=Transparent}
 [ \p{Other_Joining_Type=Deduce_From_General_Category} - [\p{gc=Mn}\p{gc=Me}\p{gc=Cf}] ] ⊆ \p{Joining_Type=Non_Joining}
 
-## Modifier_Combining_Mark stability.  Not yet testable, since MCM does not exist before 16.0.
-# NOTICE TO THE MAINTAINER: The following line is intended to fail after 16.0;
-# When it does, remove it and uncomment the line after it.
-\p{gc=Cn} = \p{U16:gc=Cn}  # ☚ REMOVE ME WHEN I FAIL…
-# … AND UNCOMMENT ME. ☛ # In \P{U-1:gc=Cn}, Modifier_Combining_Mark = U-1:Modifier_Combining_Mark
+## Modifier_Combining_Mark stability.
+In \P{U-1:gc=Cn}, Modifier_Combining_Mark = U-1:Modifier_Combining_Mark
 
 ##########################
 # LineBreak property