Merge remote-tracking branch 'upstream/main' into new-era-codes

Author: robertbastian

.github/workflows/gh-pages.yml

@@ -30,6 +30,7 @@ on:
       - '.github/workflows/gh-pages.yml'
 jobs:
   build:
+    name: "Build Github Pages"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -113,6 +114,7 @@ jobs:
           echo "::endgroup::"
           echo "Now go to https://cldr-smoke.unicode.org/spec/"$(basename ${GITHUB_REF_NAME})
   deploy:
+    name: "Deploy to GitHub Pages"
     # only deploy from main branch of main repo
     if: github.repository == 'unicode-org/cldr' && github.event_name == 'push' && github.ref == 'refs/heads/main'
     environment:

.github/workflows/maven.yml

@@ -1,5 +1,10 @@
 name: cldr-mvn
 
+env:
+  ## uncomment only ONE of these
+  CLDR_CHECK_MODE: FINAL_TESTING
+  # CLDR_CHECK_MODE: BUILD
+
 on:
   push:
   pull_request:
@@ -40,22 +45,30 @@ jobs:
             ${{ runner.os }}-mavenspotless-
       - name: Check Java style
         run: mvn --file=tools/pom.xml spotless:check || (echo "Style checker failed. Formatting changes can be applied by 'mvn spotless:apply'" && exit 1)
-  build:
+      - name: Restore lychee cache
+        uses: actions/cache@v3
+        with:
+          path: .lycheecache
+          key: cache-lychee-${{ github.sha }}
+          restore-keys: cache-lychee-
+      - name: Run lychee on PathDescriptions.md
+        uses: lycheeverse/lychee-action@v1
+        with:
+            args: "-n --cache --max-cache-age 10d tools/cldr-code/src/main/resources/org/unicode/cldr/util/data/PathDescriptions.md"
+            fail: true
+            format: markdown
+            output: linkcheck.md
+      - name: Link Checker Summary PathDescriptions.md
+        run: cat linkcheck.md >> $GITHUB_STEP_SUMMARY
+  package:
+    name: "Package up CLDR"
     runs-on: ubuntu-latest
     steps:
       - name: Clone Repository
         uses: actions/checkout@v4
         with:
           lfs: false
           fetch-depth: 0  # Needed for cldr-archive
-        if: github.event.inputs.git-ref == ''
-      - name: Clone Repository (manual ref)
-        uses: actions/checkout@v4
-        with:
-          lfs: false
-          fetch-depth: 0  # Needed for cldr-archive
-          ref: ${{ github.event.inputs.git-ref }}
-        if: github.event.inputs.git-ref != ''
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
@@ -97,16 +110,14 @@ jobs:
         env:
           DATADOG_SITE: ${{ secrets.DATADOG_SITE }}
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
-      - name: Checkout CLDR archive
-        run: >
-          mkdir -v ../cldr-archive && java -jar tools/cldr-code/target/cldr-code.jar checkout-archive
       - name: Build liberty server
         run: >
           mvn -s .github/workflows/mvn-settings.xml -B  -pl cldr-apps liberty:create liberty:deploy liberty:package -Dinclude=usr --file tools/pom.xml
           -DskipTests=true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Cleanup liberty server
+        # we remove the 'expanded' version of the .war file, to save space
         run: >
           zip tools/cldr-apps/target/cldr-apps.zip  -d  wlp/usr/servers/cldr/apps/expanded/\*
       - name: Upload cldr-apps.zip
@@ -119,34 +130,63 @@ jobs:
         with:
           name: cldr-code
           path: tools/cldr-code/target/cldr-code.jar
-      - name: BOM check in .xml files
-        run: >
-          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
-          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
-          -Dexec.cleanupDaemonThreads=false
-          -Dexec.args="-q --justCheckBom exemplars seed common keyboards"
-          && echo 'No XML BOM detected'
-      - name: Schema check common/main in .xml files
-        run: >
-          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
-          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
-          -Dexec.cleanupDaemonThreads=false
-          -Dexec.args="-q --schema=common/dtd/ldml.xsd  common/main"
-          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
-      - name: Keyboard Schema Check
+  build:
+    #Cannot change the name, it's a required step.
+    #name: "Build / Unit-test"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone Repository
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+          fetch-depth: 0  # Needed for cldr-archive
+        if: github.event.inputs.git-ref == ''
+      - name: Clone Repository (manual ref)
+        uses: actions/checkout@v4
+        with:
+          lfs: false
+          fetch-depth: 0  # Needed for cldr-archive
+          ref: ${{ github.event.inputs.git-ref }}
+        if: github.event.inputs.git-ref != ''
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('tools/**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - name: Cache local npm repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+            node-
+      - name: Cache webpack
+        uses: actions/cache@v4
+        with:
+          path: tools/cldr-apps/target/webpack_cache
+          key: ${{ runner.os }}-webpack-${{ hashFiles('tools/cldr-apps/package*') }}-${{ hashFiles('tools/cldr-apps/src/**') }}
+          restore-keys: |
+            ${{ runner.os }}-webpack-
+            webpack-
+      - name: Build with Maven
         run: >
-          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
-          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
-          -Dexec.cleanupDaemonThreads=false
-          -Dexec.args="--schema=keyboards/dtd/ldmlKeyboard.xsd keyboards/3.0"
-          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
-      - name: Keyboard Test Schema Check
+          mvn -s .github/workflows/mvn-settings.xml -B compile install package --file tools/pom.xml
+          -DskipTests=true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout CLDR archive
         run: >
-          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
-          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
-          -Dexec.cleanupDaemonThreads=false
-          -Dexec.args="--schema=keyboards/dtd/ldmlKeyboardTest.xsd keyboards/test"
-          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
+          mkdir -v ../cldr-archive && java -jar tools/cldr-code/target/cldr-code.jar checkout-archive
+        env:
+          GIT_LFS_SKIP_SMUDGE: 1
       - name: Setup MySQL
         run: |
           sudo systemctl start mysql.service
@@ -163,6 +203,8 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   check:
+    #Cannot change the name, it's a required step.
+    #name: "Run ConsoleCheckCLDR"
     runs-on: ubuntu-latest
     steps:
       - name: Clone Repository
@@ -194,14 +236,102 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: run CLDR console check
-        run: java -DCLDR_GITHUB_ANNOTATIONS=true -DCLDR_DIR=$(pwd) -Xmx6g -jar tools/cldr-code/target/cldr-code.jar check -S common,seed -e -z FINAL_TESTING
+        run: java -DCLDR_GITHUB_ANNOTATIONS=true -DCLDR_DIR=$(pwd) -Xmx6g -jar tools/cldr-code/target/cldr-code.jar check -S common,seed -e -z ${CLDR_CHECK_MODE}
+      - name: BOM check in .xml files
+        run: >
+          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
+          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
+          -Dexec.cleanupDaemonThreads=false
+          -Dexec.args="-q --justCheckBom exemplars seed common keyboards"
+          && echo 'No XML BOM detected'
+      - name: Schema check common/main in .xml files
+        run: >
+          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
+          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
+          -Dexec.cleanupDaemonThreads=false
+          -Dexec.args="-q --schema=common/dtd/ldml.xsd  common/main"
+          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
+      - name: Keyboard Schema Check
+        run: >
+          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
+          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
+          -Dexec.cleanupDaemonThreads=false
+          -Dexec.args="--schema=keyboards/dtd/ldmlKeyboard.xsd keyboards/3.0"
+          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
+      - name: Keyboard Test Schema Check
+        run: >
+          mvn -s .github/workflows/mvn-settings.xml --file tools/pom.xml -pl cldr-code
+          exec:java -Dexec.mainClass=org.unicode.cldr.util.XMLValidator
+          -Dexec.cleanupDaemonThreads=false
+          -Dexec.args="--schema=keyboards/dtd/ldmlKeyboardTest.xsd keyboards/test"
+          || (echo 'Schema failed, rerun GenerateDtd' ; exit 0)
+  servertest:
+    name: "Docker test cldr-apps"
+    needs:
+      - package
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone Repository
+      uses: actions/checkout@v4
+      with:
+        lfs: false
+    - run: mkdir -p tools/cldr-apps/target
+    - name: Download cldr-apps.zip
+      uses: actions/download-artifact@v4
+      with:
+        name: cldr-apps-server
+        path: tools/cldr-apps/target/cldr-apps.zip
+    - name: Cache local Maven repository
+      uses: actions/cache@v4
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-mavencheck-${{ hashFiles('tools/**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-mavencheck-
+    - name: Cache local npm repository
+      uses: actions/cache@v4
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+          node-
+    - name: Pull Images
+      run: docker compose pull -q
+      working-directory: tools/cldr-apps
+    - name: Build Containers
+      run: docker compose build
+      working-directory: tools/cldr-apps
+    - name: Bringup Containers
+      run: docker compose up -d cldr-apps selenium
+      working-directory: tools/cldr-apps
+    - name: Build webdriver
+      run: docker compose run -v ~/.m2/repository:/root/.m2/repository:rw --rm webdriver mvn -B test-compile
+      working-directory: tools/cldr-apps
+    - name: Test with Webdriver
+      # See tools/cldr-apps/README.md
+      run: >
+        docker compose run -v ~/.m2/repository:/root/.m2/repository:rw --rm webdriver || ( docker compose exec -it cldr-apps tail /logs/messages.log ; false )
+      working-directory: tools/cldr-apps
+    - name: Prepare client tests
+      run: docker run --rm -v ~/.npm:/root/.npm:rw -v $(pwd)/js:/home/node/app:rw -w /home/node/app node npm ci
+      working-directory: tools/cldr-apps
+    - name: Run client tests against ST
+      run: docker compose run --rm client-test || ( docker compose exec -it cldr-apps tail /logs/messages.log ; false )
+      working-directory: tools/cldr-apps
+    - name: Shut down containers
+      run: docker compose stop || true
+      working-directory: tools/cldr-apps
   deploy:
+    name: "Deploy to cldr-smoke for testing"
     # don't run deploy on manual builds!
     if: github.repository == 'unicode-org/cldr' && github.event_name == 'push' && github.ref == 'refs/heads/main' && github.event.inputs.git-ref == ''
     needs:
       - build
       - check
       - formatter
+      - package
+      # - servertest
     runs-on: ubuntu-latest
     steps:
     - name: Download cldr-apps.zip

.github/workflows/site.yml

@@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
 name: CLDR Site
 
 permissions:
@@ -18,7 +19,7 @@ on:
         paths:
         #  - "tools/scripts/tr-archive/**"
          - "docs/site/**"
-         - '.github/workflows/site.yml'    
+         - '.github/workflows/site.yml'
 jobs:
   cloudflare:
     if: github.event_name == 'push' || github.event_name == 'pull_request'
@@ -28,6 +29,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         lfs: false
+        fetch-depth: 2
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: 3.2
@@ -41,15 +43,20 @@ jobs:
     - name: Pre-install Wrangler
       if: (github.event_name == 'push') || (github.event.pull_request.head.repo.full_name == 'unicode-org/cldr')
       run: npm i -g [email protected]
-      # see https://github.com/cloudflare/wrangler-action/issues/286
     - name: Deploy cldr.pages.dev
       if: (github.event_name == 'push') || (github.event.pull_request.head.repo.full_name == 'unicode-org/cldr')
       id: deploy
       uses: cloudflare/wrangler-action@v3
       with:
         apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
         accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-        command: pages deploy ./_site --project-name=cldr
+        # use '--branch pr####' (pr###.cldr.pages.dev) for PRs, otherwise goes to cldr.unicode.org (production)
+        command: pages deploy ./_site --project-name=cldr ${{ github.event_name == 'pull_request' && format('--branch pr{0}', github.event.pull_request.number) || '' }}
+    - name: Calculate changed site files
+      id: prdiff
+      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'unicode-org/cldr'
+      run: |
+        (echo -n 'prdiff=' ; git diff --name-status HEAD^ HEAD | cut -d/ -f2- | grep '^site/.*\.md$'  | sed -e 's%^site/\(.*\)\.md%\\n- [\1.md](${{ steps.deploy.outputs.pages-deployment-alias-url }}/\1)%' | tr -d '\012' ) | tee -a $GITHUB_OUTPUT
     - name: Add deploy comment to PR
       uses: actions/github-script@v7
       if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'unicode-org/cldr'
@@ -59,7 +66,7 @@ jobs:
             issue_number: context.issue.number,
             owner: context.repo.owner,
             repo: context.repo.repo,
-            body: '✨ deployed to <${{ steps.deploy.outputs.deployment-url }}>'
+            body: '✨ This commit deployed to <${{ steps.deploy.outputs.deployment-url }}> \n✨ This PR deployed to <${{ steps.deploy.outputs.pages-deployment-alias-url }}>\n\n**Changed Pages in the last commit:**\n\n${{ steps.prdiff.outputs.prdiff }}'
             })
           github.rest.issues.addLabels({
             issue_number: context.issue.number,

.node-version

@@ -1 +1 @@
-v20.18.0
+v22.14.0

bin/.gitignore

@@ -0,0 +1 @@
+/org/

bin/org/unicode/cldr/web/package.html

@@ -0,0 +1,15 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+<head>
+<!--
+Copyright (C) 2004-2007, Unicode, Inc., International Business Machines Corporation, and others.
+For terms of use, see http://www.unicode.org/terms_of_use.html
+-->
+</head>
+<body bgcolor="white">
+<h3>Terms of Use</h3>
+<pre>* All files Copyright (c) 2004-2007, Unicode Inc, and others.  All Rights Reserved.
+* For terms of use, see <a href="http://unicode.org/copyright.html#Exhibit1">http://unicode.org/copyright.html#Exhibit1</a></pre>
+<p>&nbsp;</p>
+</body>
+</html>

common/README.md

@@ -0,0 +1,5 @@
+# CLDR `common` directory
+
+This directory contains CLDR data corresponding to the release.
+
+For copyright, terms of use, and further details, see the top [README](../README.md).

common/annotations/README.md

@@ -0,0 +1,5 @@
+# CLDR `common/annotations` directory
+
+This directory contains annotations and TTS names for characters.
+
+For copyright, terms of use, and further details, see the top [README](../../README.md).