Ucdxml and TR42

.github/ISSUE_TEMPLATE/unicodedata.md

@@ -0,0 +1,14 @@
+---
+name: Feedback on Unicode data, properties, algorithms
+about: Not for reporting bugs or suggestions
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+## Members of UTC/PAG and other Unicoders with access
+Please submit an issue in the Unicode-internal PAG repo.
+
+## Otherwise
+Please submit feedback via https://www.unicode.org/reporting.html

.github/ISSUE_TEMPLATE/unicodetools.md

@@ -0,0 +1,13 @@
+---
+name: Feedback on the internal Unicode Tools code and processes
+about: Internal tools used by UTC/PAG
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+(Bugs, suggestions, ideas for internal Unicode Tools code, for its maintainers, their processes, etc.)
+
+(These tools are public and open source but not public deliverables. They are used internally by Unicode maintainers.
+Data is published at https://www.unicode.org/Public/ -- feedback: https://www.unicode.org/reporting.html)

.github/workflows/build-jsp.yml

@@ -3,19 +3,26 @@ name: Build JSP
 env:
   CURRENT_UVERSION: 17.0.0 # FIX_FOR_NEW_VERSION
   PREVIOUS_UVERSION: 16.0.0  # not used at present
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 on:
-  push:
-    branches:
-    - '*'
   pull_request:
     branches:
     - '*'
+  push:
+    tags:
+    - '*'
 jobs:
   build:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get the CLDR_REF from pom.xml
       id: cldr_ref
       run: echo "CLDR_REF="$(mvn help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
@@ -29,7 +36,7 @@ jobs:
         restore-keys: |
           cldr
     - name: Check out CLDR
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: unicode-org/cldr
         path: cldr
@@ -45,12 +52,16 @@ jobs:
       with:
         java-version: 11
     - name: Cache local Maven repository
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/.m2/repository
         key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
           ${{ runner.os }}-maven-
+    - name: Fetch Last Resort Font
+      run: |
+        wget https://github.com/unicode-org/last-resort-font/releases/latest/download/LastResort-Regular.ttf
+        mv ./LastResort-Regular.ttf ./UnicodeJsps/src/main/webapp/
     # TODO: move this DOWN after JSPs once it works
     - name: Regenerate the property cache files
       run: >
@@ -69,10 +80,46 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Upload UnicodeJsps.war
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: UnicodeJsps
         path: UnicodeJsps/target/UnicodeJsps.war
-    - name: build docker image
-      run: cd UnicodeJsps && bash update-bidic-ucd.sh && docker build .
-
+    - name: Build C Bidi
+      run: cd UnicodeJsps && bash update-bidic-ucd.sh
+    - name: Log in to the Container registry
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+      with:
+        registry: ${{ env.REGISTRY }}
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta
+      uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+    - name: Build and push Docker image
+      if: github.event_name != 'pull_request'
+      id: push
+      uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+      with:
+        context: UnicodeJsps/
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+    - name: Build and Docker image (PR)
+      if: github.event_name == 'pull_request'
+      id: builddocker
+      uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+      with:
+        context: UnicodeJsps/
+        push: false
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+    - name: Generate artifact attestation
+      if: github.event_name != 'pull_request'
+      uses: actions/attest-build-provenance@v2
+      with:
+        subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+        subject-digest: ${{ steps.push.outputs.digest }}
+        push-to-registry: true

.github/workflows/cache_retain.yml

@@ -30,6 +30,10 @@ jobs:
   retain-maven-cache:
     name: Run all tests with Maven
     runs-on: ubuntu-latest
+    # Only run this on the upstream repo. Otherwise, running in a personal fork will cause
+    # Github to disable the personal fork copy of the workflow
+    # (Github complains about running a scheduled workflow on a repo with > 60 days of inactivity)
+    if: github.ref == 'refs/heads/main' && github.repository == 'unicode-org/unicodetools'
     steps:
       - name: Checkout and setup
         uses: actions/checkout@v2

.github/workflows/cli-build-instructions.yml

@@ -176,7 +176,7 @@ jobs:
 
       - name: Upload UnicodeTestResults artifact
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: unicode-test-results
           path: |

.github/workflows/push-jsp-on-tag.yml

@@ -49,7 +49,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Upload UnicodeJsps.war
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: UnicodeJsps
         path: UnicodeJsps/target/UnicodeJsps.war

.gitignore

@@ -43,6 +43,7 @@ perf-*.xml
 test-*.xml
 
 # Directories
+.idea/
 .settings/
 .vs/
 .vscode/

UnicodeJsps/pom.xml

@@ -31,11 +31,6 @@
 			<groupId>com.ibm.icu</groupId>
 			<artifactId>icu4j</artifactId>
 		</dependency>
-
-		<dependency>
-			<groupId>com.ibm.icu</groupId>
-			<artifactId>utilities-for-cldr</artifactId>
-		</dependency>
 		<!-- cldr -->
 		<dependency>
 			<groupId>org.unicode.cldr</groupId>

UnicodeJsps/src/main/java/org/unicode/jsp/Annotations.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.text.UnicodeSet;
 import java.util.Collections;

UnicodeJsps/src/main/java/org/unicode/jsp/BidiCharMap.java

@@ -1,7 +1,7 @@
 /** */
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
 import com.ibm.icu.text.UnicodeSet;

UnicodeJsps/src/main/java/org/unicode/jsp/Confusables.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.text.Normalizer;
 import com.ibm.icu.text.Normalizer.Mode;

UnicodeJsps/src/main/java/org/unicode/jsp/CreateInversions.java

@@ -1,7 +1,7 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
-import com.ibm.icu.dev.util.UnicodeMapIterator;
+import com.ibm.icu.impl.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMapIterator;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.text.UnicodeSet;
 import com.ibm.icu.text.UnicodeSetIterator;

UnicodeJsps/src/main/java/org/unicode/jsp/NFM.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import java.util.regex.Pattern;
 

UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
 import com.ibm.icu.lang.UScript;

UnicodeJsps/src/main/java/org/unicode/jsp/ScriptTester2.java

@@ -4,7 +4,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.TreeMultimap;
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.lang.CharSequences;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.text.Normalizer2;

UnicodeJsps/src/main/java/org/unicode/jsp/SequenceData.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.lang.CharSequences;
 import com.ibm.icu.text.UnicodeSet;

UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeDataInput.java

@@ -1,6 +1,6 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.text.UnicodeSet;
 import java.io.DataInput;
 import java.io.IOException;

UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeJsp.java

@@ -166,6 +166,19 @@ public static String getSimpleSet(
         String a_out;
         a.clear();
         try {
+            if (setA.length() > 4
+                    && setA.startsWith("[:k")
+                    && setA.endsWith(":]")
+                    && setA.contains("=")
+                    && !setA.substring(2, setA.length() - 2).contains(":]")
+                    && XPropertyFactory.make()
+                            .getProperty(setA.substring(2, setA.indexOf("=")))
+                            .isMultivalued()) {
+                throw new Exception(
+                        "POSIX-style queries for multivalued Unihan properties are temporarily disabled.  Try \\p{"
+                                + setA.substring(2, setA.length() - 2)
+                                + "}");
+            }
             // setA = UnicodeSetUtilities.MyNormalize(setA, Normalizer.NFC);
             a.addAll(UnicodeSetUtilities.parseUnicodeSet(setA));
             a_out = UnicodeUtilities.getPrettySet(a, abbreviate, escape);

UnicodeJsps/src/main/java/org/unicode/jsp/UnicodeUtilities.java

@@ -1,7 +1,7 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.impl.Row.R4;
+import com.ibm.icu.impl.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.lang.UCharacter;
 import com.ibm.icu.lang.UProperty;
@@ -1648,6 +1648,7 @@ class PropertyAssignment {
             VersionInfo last;
             String value;
         }
+        final boolean isMultivalued = getFactory().getProperty(propName).isMultivalued();
         List<PropertyAssignment> history = new ArrayList<>();
         // TODO(eggrobin): TUP normalization chokes on sufficiently old versions, but this is not
         // worth debugging as we want to get rid of it.
@@ -1732,17 +1733,21 @@ class PropertyAssignment {
                 out.append(
                         "<td"
                                 + defaultClass
-                                + "><a target='u' "
-                                + (isNew ? "class='changed' " : "")
-                                + "href='list-unicodeset.jsp?a=[:"
-                                + (isCurrent ? "" : "U" + last + ":")
-                                + propName
-                                + "="
-                                + hValue
-                                + ":]'>"
+                                + ">"
+                                + (isMultivalued
+                                        ? ""
+                                        : ("<a target='u' "
+                                                + (isNew ? "class='changed' " : "")
+                                                + "href='list-unicodeset.jsp?a=[:"
+                                                + (isCurrent ? "" : "U" + last + ":")
+                                                + propName
+                                                + "="
+                                                + hValue
+                                                + ":]'>"))
                                 + versionRange
                                 + hValue
-                                + "</a></td>");
+                                + (isMultivalued ? "" : "</a>")
+                                + "</td>");
             }
         }
         out.append("</tr>");