Merge branch 'main' into 204_siebzehn_leibnizsche-Zeichen

Author: jowilco

.github/workflows/build-jsp.yml

@@ -1,8 +1,8 @@
 name: Build JSP
 
 env:
-  CURRENT_UVERSION: 17.0.0 # FIX_FOR_NEW_VERSION
-  PREVIOUS_UVERSION: 16.0.0  # not used at present
+  CURRENT_UVERSION: 18.0.0 # FIX_FOR_NEW_VERSION
+  PREVIOUS_UVERSION: 17.0.0  # not used at present
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 

.github/workflows/cli-build-instructions.yml

@@ -9,8 +9,8 @@ on:
     - '*'
 
 env:
-  CURRENT_UVERSION: 17.0.0  # FIX_FOR_NEW_VERSION
-  PREVIOUS_UVERSION: 16.0.0
+  CURRENT_UVERSION: 18.0.0  # FIX_FOR_NEW_VERSION
+  PREVIOUS_UVERSION: 17.0.0
 
 jobs:
 

UnicodeJsps/src/main/java/org/unicode/jsp/UBAVersion.java

@@ -19,7 +19,8 @@ public class UBAVersion {
                     Age_Values.V14_0,
                     Age_Values.V15_0,
                     Age_Values.V15_1,
-                    Age_Values.V16_0
+                    Age_Values.V16_0,
+                    Age_Values.V17_0
                     // FIX_FOR_NEW_VERSION
                     /* Current version is always last */
                     );

UnicodeJsps/src/test/java/org/unicode/jsptest/TestAll.java

@@ -2,7 +2,7 @@
 
 package org.unicode.jsptest;
 
-import com.ibm.icu.dev.test.TestFmwk.TestGroup;
+import org.unicode.cldr.icu.dev.test.TestFmwk.TestGroup;
 
 /** Top level test used to run all other tests as a batch. */
 public class TestAll extends TestGroup {

UnicodeJsps/src/test/java/org/unicode/jsptest/TestGenerate.java

@@ -1,6 +1,5 @@
 package org.unicode.jsptest;
 
-import com.ibm.icu.dev.test.TestFmwk;
 import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.lang.UCharacter;
@@ -15,6 +14,7 @@
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.TreeSet;
+import org.unicode.cldr.icu.dev.test.TestFmwk;
 import org.unicode.cldr.util.props.BagFormatter;
 import org.unicode.cldr.util.props.UnicodeLabel;
 import org.unicode.idna.Idna;

UnicodeJsps/src/test/java/org/unicode/jsptest/TestMultivalued.java

@@ -96,21 +96,21 @@ public void TestEmpty() {
         assertEquals("exemplar(0x0000)", "", exemplarProp.getValue(0x0000));
         assertEquals("exemplar(α)", "el", exemplarProp.getValue('α'));
 
-        UnicodeSet exem = UnicodeSetUtilities.parseUnicodeSet("\\p{exem}");
-        assertTrue("\\p{exem} contains 0", exem.contains(0x0000));
-        assertFalse("\\p{exem} contains α", exem.contains('α'));
+        UnicodeSet exem = UnicodeSetUtilities.parseUnicodeSet("\\p{exem=}");
+        assertTrue("\\p{exem=} contains 0", exem.contains(0x0000));
+        assertFalse("\\p{exem=} contains α", exem.contains('α'));
         UnicodeSet exem3 = UnicodeSetUtilities.parseUnicodeSet("\\p{exem=el}");
         assertFalse("\\p{exem=el} contains 0", exem3.contains(0x0000));
         assertTrue("\\p{exem=el} contains α", exem3.contains('α'));
 
-        String unicodeSetString = "[\\p{Greek}&\\p{exem}]";
+        String unicodeSetString = "[\\p{Greek}&\\p{exem=}]";
         UnicodeSet parsed = UnicodeSetUtilities.parseUnicodeSet(unicodeSetString);
 
         String first = parsed.iterator().next();
         String firstValue = exemplarProp.getValue(first.codePointAt(0));
         assertEquals(unicodeSetString, "", firstValue);
 
-        String unicodeSetString2 = "[\\p{Greek}&\\P{exem}]";
+        String unicodeSetString2 = "[\\p{Greek}&\\P{exem=}]";
         UnicodeSet parsed2 = UnicodeSetUtilities.parseUnicodeSet(unicodeSetString2);
 
         String first2 = parsed2.iterator().next();

UnicodeJsps/src/test/java/org/unicode/jsptest/TestTypology.java

@@ -1,6 +1,5 @@
 package org.unicode.jsptest;
 
-import com.ibm.icu.dev.test.TestFmwk;
 import com.ibm.icu.impl.Row;
 import com.ibm.icu.impl.Row.R2;
 import com.ibm.icu.impl.Row.R3;
@@ -23,6 +22,7 @@
 import java.util.TreeSet;
 import java.util.regex.Pattern;
 import org.unicode.cldr.draft.FileUtilities;
+import org.unicode.cldr.icu.dev.test.TestFmwk;
 import org.unicode.cldr.util.MultiComparator;
 import org.unicode.cldr.util.UnicodeSetPrettyPrinter;
 import org.unicode.jsp.Typology;

docs/build.md

@@ -10,20 +10,19 @@ can be used to:
 *   build 4 chart folders on the unicode site.
 *   build files for ICU (collation, NFSkippable)
 
-**WARNING!!**
-
-*   This is NOT production level code, and should never be used in programs.
-*   The API is subject to change without notice, and will not be maintained.
-*   The source is uncommented, and has many warts; since it is not production
-    code, it has not been worth the time to clean it up.
-*   It will probably need some adjustments on Unix or Windows, such as changing
-    the file separator.
-*   Currently it uses hard-coded directory names.
-*   The contents of multiple versions of the UCD must be copied to a local
-    directory, as described below.
-*   ***It will be useful to look at the history of the files in git to see the
-    kinds of rule changes that are made!***
-    *   Unfortunately, we lost some change history of about 1.5 years(?) leading up to April 2020.
+> [!CAUTION]
+> *   This is NOT production level code, and should never be used in programs.
+> *   The API is subject to change without notice, and will not be maintained.
+> *   The source is uncommented, and has many warts; since it is not production
+>     code, it has not been worth the time to clean it up.
+> *   It will probably need some adjustments on Unix or Windows, such as changing
+>     the file separator.
+> *   Currently it uses hard-coded directory names.
+> *   The contents of multiple versions of the UCD must be copied to a local
+>     directory, as described below.
+> *   ***It will be useful to look at the history of the files in git to see the
+>     kinds of rule changes that are made!***
+>     *   Unfortunately, we lost some change history of about 1.5 years(?) leading up to April 2020.
 
 ## Instructions
 
@@ -86,6 +85,31 @@ Currently, some tests run on the generated output files of a tool (ex: in order
 | UNICODETOOLS_GEN_DIR    | /usr/local/google/home/mscherer/unitools/mine/Generated        |
 | UVERSION                | 14.0.0                                                         |
 
+#### Github Maven artifacts and authentication
+
+If you have not correctly configured your local settings file according to <http://cldr.unicode.org/development/maven>,
+then you will likely see error messages when your Maven build tries to access (including just reading)
+the Maven dependency artifacts such as:
+
+```
+[ERROR] Failed to execute goal on project unicodetools-testutils: Could not collect dependencies for project org.unicode.unicodetools:unicodetools-testutils:jar:1.0.0
+[ERROR] Failed to read artifact descriptor for com.ibm.icu:icu4j:jar:78.0.1-SNAPSHOT
+[ERROR]         Caused by: The following artifacts could not be resolved: com.ibm.icu:icu4j:pom:78.0.1-20250916.173842-8 (present, but unavailable): Could not transfer artifact com.ibm.icu:icu4j:pom:78.0.1-20250916.173842-8 from/to github (https://maven.pkg.github.com/unicode-org/cldr): authentication failed for https://maven.pkg.github.com/unicode-org/cldr/com/ibm/icu/icu4j/78.0.1-SNAPSHOT/icu4j-78.0.1-20250916.173842-8.pom, status: 401 Unauthorized
+[ERROR] Failed to read artifact descriptor for org.unicode.cldr:cldr-code:jar:0.0.0-SNAPSHOT-3404124632
+[ERROR]         Caused by: The following artifacts could not be resolved: org.unicode.cldr:cldr-code:pom:0.0.0-SNAPSHOT-3404124632 (present, but unavailable): Could not transfer artifact org.unicode.cldr:cldr-code:pom:0.0.0-SNAPSHOT-3404124632 from/to github (https://maven.pkg.github.com/unicode-org/cldr): authentication failed for https://maven.pkg.github.com/unicode-org/cldr/org/unicode/cldr/cldr-code/0.0.0-SNAPSHOT-3404124632/cldr-code-0.0.0-SNAPSHOT-3404124632.pom, status: 401 Unauthorized
+```
+
+This happens because [Github's Maven artifact registry requires authentication](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-apache-maven-registry), 
+via a Github token, even when you are only reading artifacts.
+
+> [!IMPORTANT]
+> If you run into this issue, then:
+> 
+> 1. revisit the [CLDR Maven setup link](http://cldr.unicode.org/development/maven)
+to ensure that you have created the correct type of Github access token: classic, _not_ fine-grained—with a fine-grained token, you will get a 403 rather than a 401—; and that you have granted the correct permissions to it.
+> 2. if you are copy-and-pasting any of the example Unicode Tools task Maven commands from the instructions below and/or from the CI workflow files,
+then you must *remove* the `-s .github/workflows/mvn-settings.xml` that is needed only for CI.
+Removing the `-s` option will use the settings in your default local settings file at `~/.m2/settings.xml`.
 
 ### Editing and tool execution setup
 

docs/egg.md

@@ -0,0 +1,42 @@
+This is the procedure used by eggrobin to build and deploy the JSPs from his
+Windows machines. Some of the commands must be run in PowerShell, others in the
+wsl bash.
+
+WSL:
+```bash
+pushd UnicodeJsps
+./update-bidic-ucd.sh
+popd
+wget https://github.com/unicode-org/last-resort-font/releases/latest/download/LastResort-Regular.ttf
+mv ./LastResort-Regular.ttf ./UnicodeJsps/src/main/webapp/
+```
+Powershell:
+```powershell
+mvn -B package -am -pl UnicodeJsps -DskipTests=true
+$CLDR_REF = $(mvn help:evaluate "-Dexpression=cldr.version" -q -DforceStdout).Split("-")[2]
+if (-not (Test-Path cldr)) {
+  git clone https://github.com/unicode-org/cldr.git "--reference=..\cldr"
+}
+Push-Location cldr
+git fetch
+git reset --hard $CLDR_REF
+Pop-Location
+```
+WSL:
+```bash
+mkdir -p UnicodeJsps/target && tar -cpz --exclude=.git -f UnicodeJsps/target/cldr-unicodetools.tgz ./cldr/ ./unicodetools/
+```
+Powershell:
+```powershell
+rm .\Generated\* -recurse -force;
+mvn compile exec:java '-Dexec.mainClass="org.unicode.jsp.RebuildPropertyCache"' -am -pl unicodetools   "-DUNICODETOOLS_GEN_DIR=Generated"  "-DUNICODETOOLS_REPO_DIR=." "-DCLDR_DIR=..\cldr\"
+```
+WSL:
+```bash
+tar -cpz -f UnicodeJsps/target/generated.tgz ./Generated/
+docker build -t us-central1-docker.pkg.dev/goog-unicode-dev/unicode-jsps/unicode-jsps:latest UnicodeJsps/
+```
+Powershell:
+```powershell
+docker push us-central1-docker.pkg.dev/goog-unicode-dev/unicode-jsps/unicode-jsps:latest
+```

docs/pipeline.md

@@ -164,13 +164,13 @@ git merge main
 git checkout main unicodetools/data/ucd/dev/Derived*
 git checkout main unicodetools/data/ucd/dev/extracted/*
 git checkout main unicodetools/data/ucd/dev/auxiliary/*
-rm -r ../Generated/BIN/17.0.0.0/
-rm -r ../Generated/BIN/UCD_Data17.0.0.bin
-mvn -s ~/.m2/settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version 17.0.0 build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=17.0.0
+rm -r ../Generated/BIN/18.0.0.0/
+rm -r ../Generated/BIN/UCD_Data18.0.0.bin
+mvn -s ~/.m2/settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCD.Main"  -Dexec.args="version 18.0.0 build MakeUnicodeFiles" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=18.0.0
 # fix merge conflicts in unicodetools/src/main/java/org/unicode/text/UCD/UCD_Types.java
 #   and in UCD_Names.java
 # rerun mvn
-cp -r ../Generated/UCD/17.0.0/* unicodetools/data/ucd/dev
+cp -r ../Generated/UCD/18.0.0/* unicodetools/data/ucd/dev
 rm unicodetools/data/ucd/dev/ZZZ-UNCHANGED-*
 rm unicodetools/data/ucd/dev/*/ZZZ-UNCHANGED-*
 rm unicodetools/data/ucd/dev/extra/*
@@ -195,7 +195,7 @@ eggrobin (Windows, in-source).
 ```powershell
 rm .\Generated\* -recurse -force
 mvn compile exec:java '-Dexec.mainClass="org.unicode.text.UCD.Main"'  '-Dexec.args="build MakeUnicodeFiles"' -am -pl unicodetools  "-DCLDR_DIR=..\cldr\"  "-DUNICODETOOLS_GEN_DIR=Generated"  "-DUNICODETOOLS_REPO_DIR=."
-cp .\Generated\UCD\17.0.0\* .\unicodetools\data\ucd\dev -recurse -force
+cp .\Generated\UCD\18.0.0\* .\unicodetools\data\ucd\dev -recurse -force
 rm unicodetools\data\ucd\dev\zzz-unchanged-*
 rm unicodetools\data\ucd\dev\*\zzz-unchanged-*
 rm .\unicodetools\data\ucd\dev\extra\*
@@ -211,7 +211,7 @@ eggrobin (Windows, in-source).
 ```powershell
 rm .\Generated\* -recurse -force
 mvn compile exec:java '-Dexec.mainClass="org.unicode.text.UCD.Main"'  '-Dexec.args="build MakeUnicodeFiles"' -am -pl unicodetools  "-DCLDR_DIR=..\cldr\"  "-DUNICODETOOLS_GEN_DIR=Generated"  "-DUNICODETOOLS_REPO_DIR=."
-cp .\Generated\UCD\17.0.0\LineBreak.txt .\unicodetools\data\ucd\dev
+cp .\Generated\UCD\18.0.0\LineBreak.txt .\unicodetools\data\ucd\dev
 ```
 
 ### GenerateEnums
@@ -229,6 +229,6 @@ git commit -m GenerateEnums
 
 eggrobin (Windows, in-source).
 ```powershell
-mvn test -am -pl unicodetools "-DCLDR_DIR=$(gl|split-path -parent)\cldr\"  "-DUNICODETOOLS_GEN_DIR=$(gl|split-path -parent)\unicodetools\Generated\"  "-DUNICODETOOLS_REPO_DIR=$(gl|split-path -parent)\unicodetools\" "-DUVERSION=17.0.0" "-Dtest=TestTestUnicodeInvariants#testAdditionComparisons" -DfailIfNoTests=false -DtrimStackTrace=false
+mvn test -am -pl unicodetools "-DCLDR_DIR=$(gl|split-path -parent)\cldr\"  "-DUNICODETOOLS_GEN_DIR=$(gl|split-path -parent)\unicodetools\Generated\"  "-DUNICODETOOLS_REPO_DIR=$(gl|split-path -parent)\unicodetools\" "-DUVERSION=18.0.0" "-Dtest=TestTestUnicodeInvariants#testAdditionComparisons" -DfailIfNoTests=false -DtrimStackTrace=false
 ```
 Results are in Generated\UnicodeTestResults-addition-comparisons-[RMG issue number].html.