Merge remote-tracking branch 'la-vache/main' into jh-tolong-siki

Author: eggrobin

.github/workflows/build-jsp.yml

@@ -1,8 +1,8 @@
 name: Build JSP
 
 env:
-  CURRENT_UVERSION: 16.0.0
-  PREVIOUS_UVERSION: 15.1.0  # not used at present
+  CURRENT_UVERSION: 17.0.0 # FIX_FOR_NEW_VERSION
+  PREVIOUS_UVERSION: 16.0.0  # not used at present
 
 on:
   push:
@@ -69,7 +69,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Upload UnicodeJsps.war
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: UnicodeJsps
         path: UnicodeJsps/target/UnicodeJsps.war

.github/workflows/cli-build-instructions.yml

@@ -9,8 +9,8 @@ on:
     - '*'
 
 env:
-  CURRENT_UVERSION: 16.0.0
-  PREVIOUS_UVERSION: 15.1.0
+  CURRENT_UVERSION: 17.0.0  # FIX_FOR_NEW_VERSION
+  PREVIOUS_UVERSION: 16.0.0
 
 jobs:
 
@@ -88,7 +88,7 @@ jobs:
       # change anything, which makes little sense; but that is the job of the
       # other job.
       - name: Run invariant tests
-        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
+        run: MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testUnicodeInvariants -DfailIfNoTests=false -DCLDR_DIR=$(cd ../cldr ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd Generated; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -170,7 +170,7 @@ jobs:
       - name: Run command - Build and Test
         run: |
           cd unicodetools/mine/src
-          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml package -Dtest=!TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION  -DEMIT_GITHUB_ERRORS
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -275,6 +275,22 @@ jobs:
         with:
           repository: unicode-org/unicodetools
           path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
       - name: Get the CLDR_REF from pom.xml
         id: cldr_ref
         run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
@@ -316,9 +332,92 @@ jobs:
       - name: Run command - UCA - collation validity log
         run: |
           cd unicodetools/mine/src
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then set +e
+          fi
           # invoke main() in class ...UCA.Main
           mvn -s .github/workflows/mvn-settings.xml compile exec:java -Dexec.mainClass="org.unicode.text.UCA.Main"  -Dexec.args="writeCollationValidityLog ICU" -am -pl unicodetools  -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION
           # check for output file
           compgen -G "../Generated/UCA/*/CheckCollationValidity.html"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # Out-of-source build.
+  security:
+    name: Check security data invariants
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Unicode Tools
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/unicodetools
+          path: unicodetools/mine/src
+      - name: Checkout base UnicodeData.txt
+        if: ${{ github.event_name == 'pull_request'}}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          sparse-checkout: unicodetools/data/ucd/dev/UnicodeData.txt
+      - name: Compare repertoire
+        if: ${{ github.event_name == 'pull_request'}}
+        run: |
+          # Look for changes affecting the first two fields of UnicodeData.txt (code point and name).
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' unicodetools/mine/src/unicodetools/data/ucd/dev/UnicodeData.txt > merged-repertoire.txt
+          sed 's/^\([^;]*;[^;]*\);.*$/\1/' base/unicodetools/data/ucd/dev/UnicodeData.txt > base-repertoire.txt
+          set +e
+          diff base-repertoire.txt merged-repertoire.txt
+          echo "REPERTOIRE_CHANGED=$?" >> "$GITHUB_ENV"
+      - name: Get the CLDR_REF from pom.xml
+        id: cldr_ref
+        run: echo "CLDR_REF="$(mvn --file unicodetools/mine/src/pom.xml help:evaluate -Dexpression=cldr.version -q -DforceStdout | cut -d- -f3) >> $GITHUB_OUTPUT && cat ${GITHUB_OUTPUT}
+      - name: Verify CLDR checkout ref
+        run: echo CLDR_REF="${{ steps.cldr_ref.outputs.CLDR_REF }}" && [ "${{ steps.cldr_ref.outputs.CLDR_REF }}x" != "x" ] # fail if empty
+      - name: Cache CLDR repository
+        uses: actions/cache@v3
+        with:
+          path: cldr/mine/src
+          key: cldr-${{ steps.cldr_ref.outputs.CLDR_REF }}
+          restore-keys: |
+            cldr
+      - name: Check out CLDR
+        uses: actions/checkout@v3
+        with:
+          repository: unicode-org/cldr
+          path: cldr/mine/src
+          ref: main
+          fetch-depth: 0
+      - name: Switch CLDR to CLDR_REF
+        run: cd cldr/mine/src && git fetch && git checkout ${{ steps.cldr_ref.outputs.CLDR_REF }}
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Set up out-of-source output dir
+        run: |
+          mkdir -p unicodetools/mine/Generated/BIN
+
+      - name: Run invariant tests
+        run: |
+          cd unicodetools/mine/src
+          echo "REPERTOIRE_CHANGED=$REPERTOIRE_CHANGED"
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then ERROR="::notice"
+          else ERROR="::error"
+          fi
+          MAVEN_OPTS="-ea" mvn -s .github/workflows/mvn-settings.xml test -am -pl unicodetools -Dtest=TestTestUnicodeInvariants#testSecurityInvariants -DCLDR_DIR=$(cd ../../../cldr/mine/src ; pwd)  -DUNICODETOOLS_GEN_DIR=$(cd ../Generated ; pwd)  -DUNICODETOOLS_REPO_DIR=$(pwd)  -DUVERSION=$CURRENT_UVERSION -DfailIfNoTests=false -DEMIT_GITHUB_ERRORS 2>&1 | sed "s/^::error/$ERROR/"
+          STATUS=${PIPESTATUS[0]}
+          if [[ ${REPERTOIRE_CHANGED:-0} -ne 0 ]]
+          then exit 0
+          else exit $STATUS
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pipeline.yml

@@ -53,7 +53,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         sparse-checkout: py/pipeline-workflow
-    - name: Check L2 document
+    - name: Check L2 document and WG references
       run: |
         python3 py/pipeline-workflow/check-l2-document.py
   utc-decision:

.github/workflows/push-jsp-on-tag.yml

@@ -49,7 +49,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - name: Upload UnicodeJsps.war
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: UnicodeJsps
         path: UnicodeJsps/target/UnicodeJsps.war

.github/workflows/pythonpackage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
     - uses: actions/checkout@v3

README.md

@@ -25,6 +25,6 @@ The tools maintainers use GH issues for issues with the code in this repo.
 
 Copyright © 2001-2024 Unicode, Inc. Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the United States and other countries.
 
-The project is released under [LICENSE](./LICENSE).
-
 A CLA is required to contribute to this project - please refer to the [CONTRIBUTING.md](https://github.com/unicode-org/.github/blob/main/.github/CONTRIBUTING.md) file (or start a Pull Request) for more information.
+
+The contents of this repository are governed by the Unicode [Terms of Use](https://www.unicode.org/copyright.html) and are released under [LICENSE](./LICENSE).

UnicodeJsps/pom.xml

@@ -29,7 +29,7 @@
 		<!-- icu -->
 		<dependency>
 			<groupId>com.ibm.icu</groupId>
-			<artifactId>icu4j-for-cldr</artifactId>
+			<artifactId>icu4j</artifactId>
 		</dependency>
 
 		<dependency>

UnicodeJsps/src/main/java/org/unicode/cldr/tool/TablePrinter.java

@@ -451,6 +451,7 @@ private int findIdentical(Comparable[][] sortedFlat, int rowIndex, int colIndex)
         }
         return sortedFlat.length - rowIndex;
     }
+
     // to-do: prevent overlap when it would cause information to be lost.
     private BitSet breaksSpans = new BitSet();
 

UnicodeJsps/src/main/java/org/unicode/jsp/Confusables.java

@@ -1,6 +1,5 @@
 package org.unicode.jsp;
 
-import com.ibm.icu.dev.util.CollectionUtilities;
 import com.ibm.icu.dev.util.UnicodeMap;
 import com.ibm.icu.impl.Utility;
 import com.ibm.icu.text.Normalizer;
@@ -41,7 +40,7 @@ public static UnicodeMap<String> getMap() {
         UnicodeMap<String> result = new UnicodeMap<String>();
         for (String s : equivalents) {
             Set<String> others = new TreeSet<String>(equivalents.getEquivalences(s));
-            String list = "\u2051" + CollectionUtilities.join(others, "\u2051") + "\u2051";
+            String list = "\u2051" + String.join("\u2051", others) + "\u2051";
             for (String other : others) {
                 result.put(other, list);
             }

UnicodeJsps/src/main/java/org/unicode/jsp/Globe.java

@@ -495,6 +495,7 @@ private static void loadSourceMap(String sname) {
         }
         changeImage(frame);
     }
+
     /**
      * @param frame
      */
@@ -704,12 +705,14 @@ public Shape getClip() {
             }
             return clip;
         }
+
         /**
          * @return
          */
         public boolean usesOrigin() {
             return false;
         }
+
         // must set before use
         Transform set(double srcW, double srcH, double dstW, double dstH) {
             this.srcW = srcW;
@@ -724,6 +727,7 @@ Transform set(double srcW, double srcH, double dstW, double dstH) {
             clip = null;
             return this;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.
         // cos(-angle) =  cos(angle)
@@ -750,6 +754,7 @@ public final boolean transform(double x, double y, DPoint retcoord) {
             retcoord.y = lat_dstH.map(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= dstW && retcoord.y >= 0 && retcoord.y <= dstH;
         }
+
         // Remember that the coordinate system is upside down so apply
         // the transform as if the angle were negated.  Since inverting
         // the transform is also the same as negating the angle, itransform
@@ -774,18 +779,21 @@ public final boolean itransform(double x, double y, DPoint retcoord) {
             retcoord.y = srcH_lat.back(retcoord.y);
             return retcoord.x >= 0.0 && retcoord.x <= srcW && retcoord.y >= 0 && retcoord.y <= srcH;
         }
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _transform(DPoint retcoord);
+
         /**
          * @param input and output: latitude in y (radians from -pi/2 to pi/2) and longitude in x
          *     (radians from -pi to pi)
          */
         protected abstract void _itransform(DPoint retcoord);
 
         protected abstract Shape _getClip();
+
         /**
          * @param style
          * @return
@@ -866,6 +874,7 @@ public void _transform(DPoint retcoord) {
         public void _itransform(DPoint retcoord) {
             // nothing
         }
+
         /* (non-Javadoc)
          * @see Globe.Transform#_getClip()
          */
@@ -979,6 +988,7 @@ public static class TransformEqualAreaEllipse extends Transform {
                 debugTemp = false;
             }
         }
+
         // Area of a spherical cap is 2 pi r^2 (1-sin(lat))
         // Area of a circular segment is r^2 ( acos(p) - p sqrt(1-p^2)), where p = dist to chord/r
         // Thus we get the itransform easily:
@@ -1016,6 +1026,7 @@ public void _itransform(DPoint retcoord) {
         protected Shape _getClip() {
             return new Ellipse2D.Double(0, 0, dstW, dstH);
         }
+
         /**
          * @param in -1..1
          * @return value in 0..PI
@@ -1573,6 +1584,7 @@ public void imageComplete(int status) {
             consumer.imageComplete(status);
         }
     }
+
     /*
      public static double convertDegreesToDecimal(double degrees, double minutes, double seconds, boolean NorthOrEast) {
          double result = (degrees + minutes / 60 + seconds / 3600);
@@ -1932,6 +1944,7 @@ void fixContents() {
                 lineContents[c.yStart].add(c);
             }
         }
+
         /**
          * @param c
          * @return
@@ -2066,6 +2079,7 @@ private static void drawDegrees(
     }
 
     private static DPoint drawLineP1 = new DPoint();
+
     /*
 
      private static void drawLine(Graphics2D graphics2D, Transform trans, double x1, double y1, double x2, double y2) {
@@ -2111,6 +2125,7 @@ private static void drawLine2(Graphics2D graphics2D, Transform trans, double x,
 
     abstract static class TTransform {
         double x, y;
+
         // t is 0..1
         abstract void transform(double t);
     }
@@ -2174,11 +2189,13 @@ static class LineDrawer {
         transient double startX, startY, startT;
         // transient double endX, endY, endT;
         TTransform ttransform;
+
         // int segments = 0;
         LineDrawer(Graphics2D graphics2D, TTransform ttransform) {
             this.graphics2D = graphics2D;
             this.ttransform = ttransform;
         }
+
         // t is 0..1
         void draw(double startT, double endT) {
             this.startT = startT;