Merge remote-tracking branch 'upstream/main'

dchourasia · dchourasia · commit 568d1538389d · 2025-07-08T00:16:37.000Z
diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml
@@ -31,9 +31,10 @@ name: Build & Publish Notebook Servers (TEMPLATE)
 
 jobs:
   build:
-    runs-on: ubuntu-24.04
+    # https://docs.github.com/en/actions/how-tos/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+    runs-on: ${{ inputs.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
     env:
-      # Some pieces of code (image pulls for example) in podman consult TMPDIR or default to /var/tmp
+      # Some pieces of code (image pulls, for example) in podman consult TMPDIR or default to /var/tmp
       TMPDIR: /home/runner/.local/share/containers/tmpdir
       # Use the rootful instance of podman for sharing images with cri-o
       # https://podman-desktop.io/blog/sharing-podman-images-with-kubernetes-cluster#introduction
@@ -43,8 +44,7 @@ jobs:
       IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images"
       # GitHub image registry used for storing $(CONTAINER_ENGINE)'s cache
       CACHE: "ghcr.io/${{ github.repository }}/workbench-images/build-cache"
-      TRIVY_VERSION: 0.57.1
-      TRIVY_VULNDB: "/home/runner/.local/share/containers/trivy_db"
+      TRIVY_VERSION: 0.64.1
       # Targets (and their folder) that should be scanned using FS instead of IMAGE scan due to resource constraints
       TRIVY_SCAN_FS_JSON: '{}'
       # Makefile variables
@@ -168,17 +168,34 @@ jobs:
         run: sudo apt-get -qq remove podman crun
 
       - uses: actions/cache@v4
+        # https://docs.github.com/en/actions/reference/variables-reference#default-environment-variables
+        # https://docs.github.com/en/actions/how-tos/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables
         id: cached-linuxbrew
         with:
           path: /home/linuxbrew/.linuxbrew
-          key: linuxbrew
+          key: linuxbrew-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Install podman
-        if: steps.cached-linuxbrew.outputs.cache-hit != 'true'
+      - name: Install podman (linux/amd64)
+        if: inputs.platform == 'linux/amd64' && steps.cached-linuxbrew.outputs.cache-hit != 'true'
         run: |
           /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
           /home/linuxbrew/.linuxbrew/bin/brew install podman
 
+      # Warning: Your CPU architecture (arm64) is not supported. We only support
+      # x86_64 CPU architectures. You will be unable to use binary packages (bottles).
+      #
+      # This is a Tier 2 configuration:
+      #  https://docs.brew.sh/Support-Tiers#tier-2
+      # Do not report any issues to Homebrew/* repositories!
+      # Read the above document instead before opening any issues or PRs.
+      - name: Install podman (linux/arm64)
+        if: inputs.platform == 'linux/arm64' && steps.cached-linuxbrew.outputs.cache-hit != 'true'
+        # Error: podman: no bottle available!
+        # If you're feeling brave, you can try to install from source with:
+        run: |
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          /home/linuxbrew/.linuxbrew/bin/brew install --build-from-source podman
+
       - name: Add linuxbrew to PATH
         run: echo "/home/linuxbrew/.linuxbrew/bin/" >> $GITHUB_PATH
 
@@ -247,64 +264,6 @@ jobs:
 
       # endregion
 
-      # region Trivy init & DB pre-pull
-
-      - name: "pull_request|schedule: resolve target if Trivy scan should run"
-        id: resolve-target
-        if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
-        env:
-          EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
-          HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
-          FS_SCAN_FOLDER: ${{ fromJson(env.TRIVY_SCAN_FS_JSON)[inputs.target] }}
-        run: |
-          if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
-            if [[ -n "$FS_SCAN_FOLDER" ]]; then
-              TARGET="$FS_SCAN_FOLDER"
-              TYPE="fs"
-            else
-              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
-              TYPE="image"
-            fi
-          elif [[ "$EVENT_NAME" == "schedule" ]]; then
-            if [[ -n "$FS_SCAN_FOLDER" ]]; then
-              TARGET="$FS_SCAN_FOLDER"
-              TYPE="fs"
-            else
-              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
-              TYPE="image"
-            fi
-          fi
-
-          if [[ -n "$TARGET" ]]; then
-            echo "target=$TARGET" >> $GITHUB_OUTPUT
-            echo "type=$TYPE" >> $GITHUB_OUTPUT
-            echo "Trivy scan will run on $TARGET ($TYPE)"
-          else
-            echo "Trivy scan won't run"
-          fi
-
-      # only one db can be downloaded in one call https://github.com/aquasecurity/trivy/issues/3616
-      - name: Pre-pull Trivy vulnerabilities DB
-        if: ${{ steps.resolve-target.outputs.target }}
-        run: |
-          mkdir ${TRIVY_VULNDB}
-          podman run --rm \
-            --env PODMAN_SOCK \
-            -v ${TRIVY_VULNDB}:/cache \
-            docker.io/aquasec/trivy:$TRIVY_VERSION \
-              --cache-dir /cache \
-              image \
-              --download-db-only
-          podman run --rm \
-            --env PODMAN_SOCK \
-            -v ${TRIVY_VULNDB}:/cache \
-            docker.io/aquasec/trivy:$TRIVY_VERSION \
-              --cache-dir /cache \
-              image \
-              --download-java-db-only
-
-      # endregion
-
       # region Image build
 
       - name: Compute extra podman build args
@@ -560,6 +519,40 @@ jobs:
 
       # region Trivy vulnerability scan
 
+      - name: "pull_request|schedule: resolve target if Trivy scan should run"
+        id: resolve-target
+        if: ${{ fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'schedule' }}
+        env:
+          EVENT_NAME: ${{ fromJson(inputs.github).event_name }}
+          HAS_TRIVY_LABEL: ${{ contains(fromJson(inputs.github).event.pull_request.labels.*.name, 'trivy-scan') }}
+          FS_SCAN_FOLDER: ${{ fromJson(env.TRIVY_SCAN_FS_JSON)[inputs.target] }}
+        run: |
+          if [[ "$EVENT_NAME" == "pull_request" && "$HAS_TRIVY_LABEL" == "true" ]]; then
+            if [[ -n "$FS_SCAN_FOLDER" ]]; then
+              TARGET="$FS_SCAN_FOLDER"
+              TYPE="fs"
+            else
+              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
+              TYPE="image"
+            fi
+          elif [[ "$EVENT_NAME" == "schedule" ]]; then
+            if [[ -n "$FS_SCAN_FOLDER" ]]; then
+              TARGET="$FS_SCAN_FOLDER"
+              TYPE="fs"
+            else
+              TARGET="${{ steps.calculated_vars.outputs.OUTPUT_IMAGE }}"
+              TYPE="image"
+            fi
+          fi
+
+          if [[ -n "$TARGET" ]]; then
+            echo "target=$TARGET" >> $GITHUB_OUTPUT
+            echo "type=$TYPE" >> $GITHUB_OUTPUT
+            echo "Trivy scan will run on $TARGET ($TYPE)"
+          else
+            echo "Trivy scan won't run"
+          fi
+
       - name: Run Trivy vulnerability scanner
         if: ${{ steps.resolve-target.outputs.target }}
         run: |
@@ -588,12 +581,9 @@ jobs:
           podman run --rm \
               $PODMAN_ARGS \
               -v ${REPORT_FOLDER}:/report \
-              -v ${TRIVY_VULNDB}:/cache \
               docker.io/aquasec/trivy:$TRIVY_VERSION \
-                --cache-dir /cache \
                 $SCAN_TYPE \
                 $SCAN_ARGS \
-                --skip-db-update \
                 --scanners vuln --ignore-unfixed \
                 --exit-code 0 --timeout 30m \
                 --format template --template "@/report/$REPORT_TEMPLATE" -o /report/$REPORT_FILE \
@@ -664,7 +654,7 @@ jobs:
         # --ipc=host because Microsoft says so in Playwright docs
         # --net=host because testcontainers connects to the Reaper container's exposed port
         # we need to pass through the relevant environment variables
-        #  DEBUG configures nodejs debuggers, sets different verbosity as needed
+        #  DEBUG configures Node.js debuggers, sets different verbosity as needed
         #  CI=true is set on every CI nowadays
         #  PODMAN_SOCK should be mounted to /var/run/docker.sock, other likely mounting locations may not exist (mkdir -p)
         #  TEST_TARGET is the workbench image the test will run
diff --git a/.github/workflows/security.yaml b/.github/workflows/security.yaml
@@ -0,0 +1,35 @@
+# https://github.com/ruivieira/trustyai-explainability-python/blob/main/.github/workflows/security.yaml
+---
+name: Security
+"on":
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+jobs:
+  build:
+    name: Trivy scan (fs)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Trivy scan
+        uses: aquasecurity/trivy-action@dc5a429b52fcf669ce959baa2c2dd26090d2a6c4  # 0.32.0
+        with:
+          scan-type: 'fs'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'MEDIUM,HIGH,CRITICAL'
+          exit-code: '0'
+          ignore-unfixed: false
+
+      - name: Update Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
diff --git a/ci/cached-builds/gen_gha_matrix_jobs.py b/ci/cached-builds/gen_gha_matrix_jobs.py
@@ -21,8 +21,14 @@
 
 project_dir = pathlib.Path(__file__).parent.parent.parent.absolute()
 
+ARM64_COMPATIBLE = {
+    "codeserver-ubi9-python-3.11",
+    "codeserver-ubi9-python-3.12",
+}
+
 S390X_COMPATIBLE = {
     "runtime-minimal-ubi9-python-3.11",
+    "runtime-minimal-ubi9-python-3.12",
     # add more here
 }
 
@@ -57,6 +63,12 @@ class RhelImages(enum.Enum):
     INCLUDE_ONLY = "include-only"
 
 
+class Arm64Images(enum.Enum):
+    EXCLUDE = "exclude"
+    INCLUDE = "include"
+    ONLY = "only"
+
+
 class S390xImages(enum.Enum):
     EXCLUDE = "exclude"
     INCLUDE = "include"
@@ -82,6 +94,15 @@ def main() -> None:
         nargs="?",
         help="Whether to `include` rhel images or `exclude` them or `include-only` them",
     )
+    argparser.add_argument(
+        "--arm64-images",
+        type=Arm64Images,
+        choices=list(Arm64Images),
+        required=False,
+        default=Arm64Images.INCLUDE,
+        nargs="?",
+        help="Whether to include, exclude, or only include arm64 images",
+    )
     argparser.add_argument(
         "--s390x-images",
         type=S390xImages,
@@ -113,9 +134,12 @@ def main() -> None:
 
     targets_with_platform: list[tuple[str, str]] = []
     for target in targets:
-        if args.s390x_images != S390xImages.ONLY:
+        if args.s390x_images != S390xImages.ONLY or args.arm64_images != Arm64Images.ONLY:
             targets_with_platform.append((target, "linux/amd64"))
-        if args.s390x_images != S390xImages.EXCLUDE:
+        if args.arm64_images != Arm64Images.EXCLUDE and args.s390x_images != S390xImages.ONLY:
+            if target in ARM64_COMPATIBLE:
+                targets_with_platform.append((target, "linux/arm64"))
+        if args.s390x_images != S390xImages.EXCLUDE and args.arm64_images != Arm64Images.ONLY:
             # NOTE: hardcode the list of s390x-compatible Makefile targets in S390X_COMPATIBLE
             if target in S390X_COMPATIBLE:
                 targets_with_platform.append((target, "linux/s390x"))
diff --git a/ci/cached-builds/kubeadm.yaml b/ci/cached-builds/kubeadm.yaml
@@ -59,3 +59,16 @@ controllerManager: {}
 dns: {}
 proxy: {}
 scheduler: {}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+# ISSUE #1326: disable gc, otherwise kubelet would GC Pytorch images causing later tests to fail
+# The low threshold must be strictly less than the high threshold
+imageGCHighThresholdPercent: 100
+imageGCLowThresholdPercent: 99
+# https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#minimum-eviction-reclaim
+evictionHard:
+  # eviction threshold nodefs.available must be positive
+  nodefs.available: "1Mi"
+  # eviction threshold imagefs.available must be positive
+  imagefs.available: "1Mi"
diff --git a/codeserver/ubi9-python-3.11/Dockerfile.cpu b/codeserver/ubi9-python-3.11/Dockerfile.cpu
@@ -29,6 +29,8 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc
 ####################
 FROM base AS codeserver
 
+ARG TARGETOS TARGETARCH
+
 ARG CODESERVER_SOURCE_CODE=codeserver/ubi9-python-3.11
 ARG CODESERVER_VERSION=v4.98.0
 
@@ -50,7 +52,7 @@ WORKDIR /opt/app-root/bin
 RUN dnf install -y jq git-lfs libsndfile && dnf clean all && rm -rf /var/cache/yum
 
 # Install code-server
-RUN yum install -y "https://github.com/coder/code-server/releases/download/${CODESERVER_VERSION}/code-server-${CODESERVER_VERSION/v/}-amd64.rpm" && \
+RUN yum install -y "https://github.com/coder/code-server/releases/download/${CODESERVER_VERSION}/code-server-${CODESERVER_VERSION/v/}-${TARGETARCH}.rpm" && \
     yum -y clean all --enablerepo='*'
 
 COPY --chown=1001:0 ${CODESERVER_SOURCE_CODE}/utils utils/
