Merge pull request #643 from harshad16/sync-2024a

Sync 2024a branch with main branch

Author: harshad16

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -36,13 +36,21 @@ jobs:
 
       - name: Free up additional disk space
         # https://docs.github.com/en/actions/learn-github-actions/expressions
-        if: "${{ contains(inputs.target, 'amd') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
+        if: "${{ contains(inputs.target, 'rocm') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
          contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
         run: |
           set -x
 
           df -h
 
+          sudo apt-get update
+          sudo apt-get remove -y '^dotnet-.*'
+          sudo apt-get remove -y '^llvm-.*'
+          sudo apt-get remove -y 'php.*'
+          sudo apt-get remove -y '^mongodb-.*'
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+          sudo rm -rf /usr/local/.ghcup &
           sudo rm -rf /usr/local/lib/android &
           sudo rm -rf /usr/local/share/boost &
           sudo rm -rf /usr/local/lib/node_modules &
@@ -61,7 +69,7 @@ jobs:
           df -h
           free -h
 
-          bash ./ci/cached-builds/gha_lvm_overlay.bash
+          bash ./ci/cached-builds/gha_lvm_overlay.sh
 
           df -h
           free -h
@@ -95,10 +103,18 @@ jobs:
           podman system reset --force
           mkdir -p $HOME/.local/share/containers/storage/tmp
 
+          # start systemd user service
+          # since `brew services start podman` is buggy, let's do our own brew-compatible service
+          mkdir -p "${HOME}/.config/systemd/user/"
+          cp ci/cached-builds/homebrew.podman.service "${HOME}/.config/systemd/user/homebrew.podman.service"
+          systemctl --user daemon-reload
+          systemctl --user start homebrew.podman.service
+          echo "PODMAN_SOCK=/run/user/${UID}/podman/podman.sock" >> $GITHUB_ENV
+
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
-      - name: "push: make ${{ inputs.target }}"
+      - name: "push|schedule: make ${{ inputs.target }}"
         run: "make ${{ inputs.target }}"
-        if: "${{ fromJson(inputs.github).event_name == 'push' }}"
+        if: ${{ fromJson(inputs.github).event_name == 'push' || fromJson(inputs.github).event_name == 'schedule' }}
         env:
           IMAGE_TAG: "${{ github.ref_name }}_${{ github.sha }}"
           IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
@@ -119,5 +135,57 @@ jobs:
           IMAGE_REGISTRY: "localhost:5000/workbench-images"
           CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}"
 
+      - name: "Show podman images information"
+        run: podman images
+
+      - name: "pull_request|schedule: resolve image name if Trivy scan should run"
+        id: resolve-image
+        if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
+        env:
+          EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
+          HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
+        run: |
+          if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
+            IMAGE_NAME="localhost:5000/workbench-images:${{ inputs.target }}-${{ github.sha }}"
+            echo "image=$IMAGE_NAME" >> $GITHUB_OUTPUT
+          elif [[ "$EVENT_NAME" == "schedule" ]]; then
+            IMAGE_NAME="ghcr.io/${{ github.repository }}/workbench-images:${{ inputs.target }}-${{ github.ref_name }}_${{ github.sha }}"
+            echo "image=$IMAGE_NAME" >> $GITHUB_OUTPUT
+          fi
+
+          if [[ -z "$IMAGE_NAME" ]]; then
+            echo "Trivy scan won't run"
+          fi
+
+      - name: Run Trivy vulnerability scanner
+        if: ${{ steps.resolve-image.outputs.image }}
+        run: |
+          TRIVY_VERSION=0.53.0
+          REPORT_FOLDER=${{ github.workspace }}/report
+          REPORT_FILE=trivy-report.md
+          REPORT_TEMPLATE=trivy-markdown.tpl
+
+          mkdir -p $REPORT_FOLDER
+          cp ci/$REPORT_TEMPLATE $REPORT_FOLDER
+
+          IMAGE_NAME=${{ steps.resolve-image.outputs.image }}
+          echo "Scanning $IMAGE_NAME"
+
+          # have trivy access podman socket,
+          # https://github.com/aquasecurity/trivy/issues/580#issuecomment-666423279
+          podman run --rm \
+              -v ${PODMAN_SOCK}:/var/run/podman/podman.sock \
+              -v ${REPORT_FOLDER}:/report \
+              docker.io/aquasec/trivy:$TRIVY_VERSION \
+                image \
+                --image-src podman \
+                --podman-host /var/run/podman/podman.sock \
+                --scanners vuln --ignore-unfixed \
+                --exit-code 0 --timeout 30m \
+                --format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
+                $IMAGE_NAME
+
+          cat $REPORT_FOLDER/$REPORT_FILE >> $GITHUB_STEP_SUMMARY
+
       - run: df -h
         if: "${{ !cancelled() }}"

.github/workflows/build-notebooks-pr.yaml

@@ -1,5 +1,5 @@
 ---
-"name": "Build Notebooks"
+"name": "Build Notebooks (pr)"
 "on":
   "pull_request":
 
@@ -8,6 +8,10 @@ permissions:
   packages: read
   pull-requests: read
 
+concurrency:
+  group: ${{ format('build-notebooks-pr-{0}', github.event.pull_request.number) }}
+  cancel-in-progress: true
+
 jobs:
   gen:
     name: Generate job matrix
@@ -18,15 +22,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - run: |
+      - name: Determine targets to build based on changed files
+        run: |
+          set -x
+          git fetch --no-tags origin 'pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}'
+          git fetch --no-tags origin '+refs/heads/${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }}'
           python3 ci/cached-builds/gen_gha_matrix_jobs.py \
-            --owner=${{ github.repository_owner }} \
-            --repo=${{ github.event.pull_request.base.repo.name }} \
-            --pr-number=${{ github.event.pull_request.number }} \
-            --skip-unchanged
+            --from-ref 'origin/${{ github.event.pull_request.base.ref }}' \
+            --to-ref '${{ github.event.pull_request.head.ref }}'
         id: gen
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
 
   build:
     needs: ["gen"]

.github/workflows/build-notebooks.yaml

@@ -1,13 +1,18 @@
 ---
 # This file is autogenerated by ci/cached-builds/gen_gha_matrix_jobs.py
 {
-    "name": "Build Notebooks",
+    "name": "Build Notebooks (push)",
     "permissions": {
         "packages": "write"
     },
     "on": {
         "push": {},
-        "workflow_dispatch": {}
+        "workflow_dispatch": {},
+        "schedule": [
+            {
+                "cron": "0 2 * * *"
+            }
+        ]
     },
     "jobs": {
         "base-ubi8-python-3_8": {
@@ -74,28 +79,6 @@
             },
             "secrets": "inherit"
         },
-        "jupyter-trustyai-ubi8-python-3_8": {
-            "needs": [
-                "jupyter-datascience-ubi8-python-3_8"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "jupyter-trustyai-ubi8-python-3.8",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
-        "habana-jupyter-1_9_0-ubi8-python-3_8": {
-            "needs": [
-                "jupyter-datascience-ubi8-python-3_8"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "habana-jupyter-1.9.0-ubi8-python-3.8",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
         "habana-jupyter-1_10_0-ubi8-python-3_8": {
             "needs": [
                 "jupyter-datascience-ubi8-python-3_8"
@@ -107,17 +90,6 @@
             },
             "secrets": "inherit"
         },
-        "habana-jupyter-1_11_0-ubi8-python-3_8": {
-            "needs": [
-                "jupyter-datascience-ubi8-python-3_8"
-            ],
-            "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
-            "with": {
-                "target": "habana-jupyter-1.11.0-ubi8-python-3.8",
-                "github": "${{ toJSON(github) }}"
-            },
-            "secrets": "inherit"
-        },
         "habana-jupyter-1_13_0-ubi8-python-3_8": {
             "needs": [
                 "jupyter-datascience-ubi8-python-3_8"
@@ -444,57 +416,57 @@
             },
             "secrets": "inherit"
         },
-        "amd-c9s-python-3_9": {
+        "rocm-ubi9-python-3_9": {
             "needs": [
-                "base-c9s-python-3_9"
+                "base-ubi9-python-3_9"
             ],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
             "with": {
-                "target": "amd-c9s-python-3.9",
+                "target": "rocm-ubi9-python-3.9",
                 "github": "${{ toJSON(github) }}"
             },
             "secrets": "inherit"
         },
-        "amd-jupyter-minimal-c9s-python-3_9": {
+        "rocm-jupyter-minimal-ubi9-python-3_9": {
             "needs": [
-                "amd-c9s-python-3_9"
+                "rocm-ubi9-python-3_9"
             ],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
             "with": {
-                "target": "amd-jupyter-minimal-c9s-python-3.9",
+                "target": "rocm-jupyter-minimal-ubi9-python-3.9",
                 "github": "${{ toJSON(github) }}"
             },
             "secrets": "inherit"
         },
-        "amd-jupyter-datascience-c9s-python-3_9": {
+        "rocm-jupyter-datascience-ubi9-python-3_9": {
             "needs": [
-                "amd-jupyter-minimal-c9s-python-3_9"
+                "rocm-jupyter-minimal-ubi9-python-3_9"
             ],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
             "with": {
-                "target": "amd-jupyter-datascience-c9s-python-3.9",
+                "target": "rocm-jupyter-datascience-ubi9-python-3.9",
                 "github": "${{ toJSON(github) }}"
             },
             "secrets": "inherit"
         },
-        "amd-jupyter-tensorflow-c9s-python-3_9": {
+        "rocm-jupyter-tensorflow-ubi9-python-3_9": {
             "needs": [
-                "amd-jupyter-datascience-c9s-python-3_9"
+                "rocm-jupyter-datascience-ubi9-python-3_9"
             ],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
             "with": {
-                "target": "amd-jupyter-tensorflow-c9s-python-3.9",
+                "target": "rocm-jupyter-tensorflow-ubi9-python-3.9",
                 "github": "${{ toJSON(github) }}"
             },
             "secrets": "inherit"
         },
-        "amd-jupyter-pytorch-c9s-python-3_9": {
+        "rocm-jupyter-pytorch-ubi9-python-3_9": {
             "needs": [
-                "amd-jupyter-datascience-c9s-python-3_9"
+                "rocm-jupyter-datascience-ubi9-python-3_9"
             ],
             "uses": "./.github/workflows/build-notebooks-TEMPLATE.yaml",
             "with": {
-                "target": "amd-jupyter-pytorch-c9s-python-3.9",
+                "target": "rocm-jupyter-pytorch-ubi9-python-3.9",
                 "github": "${{ toJSON(github) }}"
             },
             "secrets": "inherit"

.github/workflows/code-quality.yaml

@@ -15,20 +15,59 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Rerun all code generators we have
-        run: python3 ci/cached-builds/gen_gha_matrix_jobs.py
+        run: bash ci/generate_code.sh
 
       - name: Check there aren't any modified files present
         run: |
-          if [[ $(git ls-files . -d -m -o --exclude-standard --full-name -v | tee modified.log | wc -l) -gt 0 ]]; then
-            echo "There are changed files"
-            exit 1
+          clean=$(git status --porcelain)
+          if [[ -z "$clean" ]]; then
+              echo "Empty git status --porcelain: $clean"
+          else
+              echo "::error::Please run 'bash ci/generate_code.sh' (the command from the previous step), commit the changed files locally, and push again."
+              echo "Uncommitted file changes detected: $clean"
+              git diff
+              exit 1
           fi
 
-      - name: Print modified files
-        if: ${{ failure() }}
-        run: |
-          cat modified.log
-          git diff
+  pytest-tests:
+    runs-on: ubuntu-latest
+    env:
+      poetry_version: '1.8.3'
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Cache poetry in ~/.local
+        uses: actions/cache/restore@v4
+        id: cache-poetry-restore
+        with:
+          path: ~/.local
+          key: "${{ runner.os }}-local-${{ env.poetry_version }}"
+
+      - name: Install poetry
+        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
+        run: pip install poetry==${{ env.poetry_version }}
+
+      - name: Save cache
+        if: steps.cache-poetry-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.local
+          key: ${{ steps.cache-poetry-restore.outputs.cache-primary-key }}
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'poetry'
+
+      - name: Configure poetry
+        run: poetry env use "${{ steps.setup-python.outputs.python-path }}"
+
+      - name: Install deps
+        run: poetry install --sync
+
+      - run: poetry run pytest
 
   code-static-analysis:
     runs-on: ubuntu-latest

.github/workflows/purge-ghcr.yaml

@@ -0,0 +1,35 @@
+---
+name: "Purge old ghcr.io test images periodically"
+
+"on":
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        type: boolean
+        default: true
+        description: "Do a dry run?"
+  schedule:
+    - cron: "0 5 * * *"  # at 05:00 every day
+
+permissions:
+  packages: write
+
+jobs:
+  clean:
+    runs-on: ubuntu-latest
+    name: Delete old test images
+    steps:
+      # https://github.com/snok/container-retention-policy?tab=readme-ov-file#parameters
+      - uses: snok/container-retention-policy@4f22ef80902ad409ed55a99dc5133cc1250a0d03  # v3.0.0
+        with:
+          # account must be the gh org name when running for an org, and 'user' when running for a user
+          account: ${{ (github.repository_owner == github.actor) && 'user' || github.repository_owner }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          image-names: |
+            ${{ github.event.repository.name }}/workbench-images
+            ${{ github.event.repository.name }}/workbench-images/build-cache
+          image-tags: "*"
+          cut-off: "3w"
+          dry-run: ${{ inputs.dry_run || false }}
+        env:
+          RUST_BACKTRACE: 1