Merge branch 'change/web/remove-old-alignment-and-transform-tokenization' into change/web/remove-tracker-analyze-state

Author: jahorton

.github/workflows/api-verification.yml

@@ -23,6 +23,7 @@ jobs:
       GIT_BRANCH: ${{ steps.environment_step.outputs.GIT_BRANCH }}
       GIT_BASE_BRANCH: ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}
       GIT_USER: ${{ steps.environment_step.outputs.GIT_USER }}
+      SKIP_API_CHECK: ${{ steps.environment_step.outputs.SKIP_API_CHECK }}
 
     steps:
     - name: Restore artifacts
@@ -40,7 +41,7 @@ jobs:
 
     - name: Set pending status on PR builds
       id: set_status
-      if: steps.environment_step.outputs.IS_TEST_BUILD == 'true'
+      if: steps.environment_step.outputs.IS_TEST_BUILD == 'true' && steps.environment_step.outputs.SKIP_API_CHECK != 'true'
       shell: bash
       run: |
         gh api \
@@ -53,6 +54,7 @@ jobs:
           -f context="$STATUS_CONTEXT"
 
     - name: Checkout
+      if: steps.environment_step.outputs.SKIP_API_CHECK != 'true'
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
       with:
         ref: '${{ steps.environment_step.outputs.GIT_SHA }}'
@@ -61,11 +63,13 @@ jobs:
         path: ${{ github.workspace }}/keyman/
 
     - name: Install devscripts
+      if: steps.environment_step.outputs.SKIP_API_CHECK != 'true'
       uses: ./keyman/.github/actions/apt-install
       with:
         packages: devscripts equivs
 
     - name: "Verify API for libkeymancore*.so (${{ steps.environment_step.outputs.GIT_BRANCH }}, branch ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}, by ${{ steps.environment_step.outputs.GIT_USER }})"
+      if: steps.environment_step.outputs.SKIP_API_CHECK != 'true'
       run: |
         echo "Verify API for libkeymancore*.so (${{ steps.environment_step.outputs.GIT_BRANCH }}, branch ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}, by ${{ steps.environment_step.outputs.GIT_USER }}):" >> $GITHUB_STEP_SUMMARY
 
@@ -79,17 +83,17 @@ jobs:
           verify 2>> $GITHUB_STEP_SUMMARY
 
     - name: Archive .symbols file
+      if: steps.environment_step.outputs.SKIP_API_CHECK != 'true' && always()
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
         name: libkeymancore.symbols
         path: ${{ github.workspace }}/keyman/linux/debian/tmp/DEBIAN/symbols
-      if: always()
 
   set_status:
     name: Set result status on PR builds
     needs: [api_verification]
     runs-on: ubuntu-latest
-    if: ${{ always() && needs.api_verification.outputs.IS_TEST_BUILD == 'true' }}
+    if: ${{ always() && needs.api_verification.outputs.IS_TEST_BUILD == 'true' && needs.api_verification.outputs.SKIP_API_CHECK != 'true' }}
     steps:
     - name: Set success
       if: needs.api_verification.result == 'success'

.github/workflows/build-test-publish-docker.yml

@@ -0,0 +1,79 @@
+# This workflow runs the build, test and publish steps for the
+# Keyman build Docker images.
+# It is triggered by a schedule and repository_dispatch event.
+
+name: "Docker images: Build, publish and test "
+run-name: "Docker images: Build, publish and test (branch ${{ github.ref_name }} on ${{ github.event_name }}/${{ github.event.action }} by ${{ github.actor }})"
+defaults:
+  run:
+    shell: bash
+
+on:
+  schedule:
+    #run every Fri, 3 pm UTC
+    - cron: '0 15 * * 5'
+
+  repository_dispatch:
+    types: ['build-test-publish-docker:*']
+
+env:
+  REGISTRY: ghcr.io
+  USERNAME: "${{ github.actor }}"
+  PASSWORD: "${{ secrets.KEYMAN_SERVER_REPO_ACCESS_PAT }}"
+
+jobs:
+
+  message:
+    name: Print message
+    runs-on: ubuntu-latest
+    if: ${{ github.event.act }}
+
+    steps:
+      - name: Print message
+        id: print-message
+        run: |
+          echo "Docker images: Build, publish and test (branch ${{ github.ref_name }} on ${{ github.event_name }}/${{ github.event.action }} by ${{ github.actor }})"
+
+  build:
+    name: Build Docker images
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Build Docker images
+        id: build
+        run: |
+          if [ -z "${{ env.USERNAME }}" ]; then
+            resources/docker-images/build.sh build: --registry ${{ env.REGISTRY }}
+          else
+            echo ${{ env.PASSWORD }} | resources/docker-images/build.sh build: --registry ${{ env.REGISTRY }} --username ${{ env.USERNAME }}
+          fi
+      - name: Publish Docker images
+        id: publish
+        run: |
+          if [ -z "${{ env.USERNAME }}" ]; then
+            resources/docker-images/build.sh publish: --registry ${{ env.REGISTRY }}
+          else
+            echo ${{ env.PASSWORD }} | resources/docker-images/build.sh publish: --registry ${{ env.REGISTRY }} --username ${{ env.USERNAME }}
+          fi
+
+  test:
+    name: Test Docker images
+    runs-on: ubuntu-latest
+    needs: build
+
+    steps:
+      - name: Checkout repository
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Test Docker images
+        id: test
+        run: |
+          if [ -z "${{ env.USERNAME }}" ]; then
+            resources/docker-images/build.sh test: --registry ${{ env.REGISTRY }}
+          else
+            echo ${{ env.PASSWORD }} | resources/docker-images/build.sh test: --registry ${{ env.REGISTRY }} --username ${{ env.USERNAME }}
+          fi

.github/workflows/cleanup-ghcr.yml

@@ -0,0 +1,45 @@
+# This workflow removes all untagged Docker images from the
+# GitHub Container Registry.
+# It is triggered by a schedule event or a manual workflow dispatch event.
+
+name: "Docker images: Cleanup GitHub Container Registry"
+run-name: "Docker images: Cleanup GitHub Container Registry (branch ${{ github.ref_name }} on ${{ github.event_name }} by ${{ github.actor }})"
+defaults:
+  run:
+    shell: bash
+
+on:
+
+  schedule:
+    #run every Fri, 6 pm UTC
+    - cron: '0 18 * * 5' #
+
+  workflow_dispatch:
+
+
+jobs:
+  message:
+    name: Print message
+    runs-on: ubuntu-latest
+    if: ${{ github.event.act }}
+
+    steps:
+      - name: Print message
+        id: print-message
+        run: |
+          echo "Docker images: Cleanup GitHub Container Registry (branch ${{ github.ref_name }} on ${{ github.event_name }} by ${{ github.actor }})"
+
+  cleanup-images:
+    name: Cleanup GitHub Container Registry images
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Removing all untagged images from GitHub Container Registry
+        id: cleanup-ghcr
+        uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16
+        with:
+          dry-run: false
+          expand-packages: true
+          packages: keyman-*-ci
+          delete-untagged: true
+          token: ${{ secrets.KEYMAN_SERVER_REPO_ACCESS_PAT }}

.github/workflows/deb-packaging.README.md

@@ -18,7 +18,8 @@ You can manually trigger a deb-packaging action on GitHub, e.g. to test changes:
       \"baseBranch\": \"master\",
       \"baseRef\": \"$(git rev-parse refs/heads/master^)\",
       \"user\": \"${USER}\",
-      \"isTestBuild\": \"true\"}" \
+      \"isTestBuild\": \"true\",
+      \"force\": \"true\"}" \
     https://api.github.com/repos/<yourgithubname>/keyman/dispatches
   ```
 
@@ -36,6 +37,7 @@ You can manually trigger a deb-packaging action on GitHub, e.g. to test changes:
       \"baseBranch\": \"master\",
       \"baseRef\": \"$(git rev-parse refs/heads/master)\",
       \"user\": \"${USER}\",
-      \"isTestBuild\": \"true\"}" \
+      \"isTestBuild\": \"true\",
+      \"force\": \"true\"}" \
     https://api.github.com/repos/<yourgithubname>/keyman/dispatches
   ```

.github/workflows/deb-packaging.yml

@@ -23,6 +23,7 @@ on:
 # baseRef:     The ref of the previous commit. For a PR the same as `baseBranch`.
 # user:        The user that triggered the build or created the PR
 # isTestBuild: false for Releases, otherwise true
+# skipApiCheck: true to skip the API check, otherwise false
 
 env:
   COLOR_GREEN: "\e[32m"
@@ -34,7 +35,7 @@ env:
 jobs:
   sourcepackage:
     name: Build source package
-    if: github.repository == 'keymanapp/keyman'
+    if: github.repository == 'keymanapp/keyman' || github.event.client_payload.force
     runs-on: ubuntu-24.04
     outputs:
       KEYMAN_VERSION: ${{ steps.version_step.outputs.KEYMAN_VERSION }}
@@ -184,28 +185,36 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    # - name: Download Artifacts
-    #   uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-    #   with:
-    #     path: artifacts
-    #     merge-multiple: true
-
-    # - name: Install dependencies
-    #   run: |
-    #     sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install autopkgtest qemu-system qemu-utils autodep8 genisoimage python3-distro-info
-
-    # - name: Build test image
-    #   run: |
-    #     cd "${GITHUB_WORKSPACE}/artifacts"
-    #     autopkgtest-buildvm-ubuntu-cloud -v --release=jammy
-
-    # - name: Run tests
-    #   run: |
-    #     cd "${GITHUB_WORKSPACE}/artifacts"
-    #     autopkgtest -B *.deb keyman_*.dsc -- qemu autopkgtest-jammy-amd64.img
-    - name: Ignore
+    - name: Download Source Artifacts
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+      with:
+        name: keyman-srcpkg
+        path: artifacts
+
+    - name: Download Binary Artifacts
+      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+      with:
+        path: artifacts
+        pattern: keyman-binarypkgs-*
+        merge-multiple: true
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install autopkgtest qemu-system qemu-utils autodep8 genisoimage python3-distro-info
+
+    - name: Build test image
+      run: |
+        cd "${GITHUB_WORKSPACE}/artifacts"
+        sudo chown ${USER} /dev/kvm
+        autopkgtest-buildvm-ubuntu-cloud -v --release=$(lsb_release -c -s)
+
+    - name: Run tests
       run: |
-        echo "Ignored for now - until working solution is in place (#13777)"
+        # Tests are defined in linux/debian/tests/test-build
+        cd "${GITHUB_WORKSPACE}/artifacts"
+        DIST="$(lsb_release -c -s)"
+        autopkgtest -B *${DIST}*.deb keyman_*.dsc -- qemu autopkgtest-${DIST}-amd64.img
 
   deb_signing:
     name: Sign source and binary packages
@@ -332,6 +341,7 @@ jobs:
         echo "GIT_BRANCH=${{ github.event.client_payload.branch }}" >> artifacts/env
         echo "GIT_BASE_BRANCH=${{ github.event.client_payload.baseBranch }}" >> artifacts/env
         echo "GIT_USER=${{ github.event.client_payload.user }}" >> artifacts/env
+        echo "SKIP_API_CHECK=${{ github.event.client_payload.skipApiCheck }}" >> artifacts/env
 
     - name: Cache artifacts
       uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3