Sync kernel-patches/vmtest

Sync workflows, configs and temporary patches with current
kernel-patches/vmtest master.

Signed-off-by: Ihor Solodrai <[email protected]>

Author: theihor

.github/scripts/download-gcc-bpf.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -euo pipefail
+
+GCC_BPF_RELEASE_GH_REPO=$1
+INSTALL_DIR=$(realpath $2)
+
+cd /tmp
+
+tag=$(gh release list -L 1 -R ${GCC_BPF_RELEASE_GH_REPO} --json tagName -q .[].tagName)
+if [[ -z "$tag" ]]; then
+    echo "Could not find latest GCC BPF release at ${GCC_BPF_RELEASE_GH_REPO}"
+    exit 1
+fi
+
+url="https://github.com/${GCC_BPF_RELEASE_GH_REPO}/releases/download/${tag}/${tag}.tar.zst"
+echo "Downloading $url"
+wget -q "$url"
+
+tarball=${tag}.tar.zst
+dir=$(tar tf $tarball | head -1 || true)
+
+echo "Extracting $tarball ..."
+tar -I zstd -xf $tarball && rm -f $tarball
+
+rm -rf $INSTALL_DIR
+mv -v $dir $INSTALL_DIR
+
+cd -
+

.github/scripts/tmpfsify-workspace.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x -euo pipefail
+
+TMPFS_SIZE=20 # GB
+MEM_TOTAL=$(awk '/MemTotal/ {print int($2/1024)}' /proc/meminfo)
+
+# sanity check: total mem is at least double TMPFS_SIZE
+if [ $MEM_TOTAL -lt $(($TMPFS_SIZE*1024*2)) ]; then
+    echo "tmpfsify-workspace.sh: will not allocate tmpfs, total memory is too low (${MEM_TOTAL}MB)"
+    exit 0
+fi
+
+dir="$(basename "$GITHUB_WORKSPACE")"
+cd "$(dirname "$GITHUB_WORKSPACE")"
+mv "${dir}" "${dir}.backup"
+mkdir "${dir}"
+sudo mount -t tmpfs -o size=${TMPFS_SIZE}G tmpfs "${dir}"
+rsync -a "${dir}.backup/" "${dir}"
+cd -
+

.github/workflows/gcc-bpf.yml

@@ -25,14 +25,19 @@ on:
 jobs:
   test:
     name: GCC BPF
-    runs-on: ${{ fromJSON(inputs.runs_on) }}
-    timeout-minutes: 100
+    runs-on: >-
+      ${{
+          contains(fromJSON(inputs.runs_on), 'codebuild')
+          && format('codebuild-bpf-ci-{0}-{1}', github.run_id, github.run_attempt)
+          || fromJSON(inputs.runs_on)
+      }}
     env:
       ARCH: ${{ inputs.arch }}
-      BPF_GCC_INSTALL_DIR: ${{ github.workspace }}/gcc-bpf
       BPF_NEXT_BASE_BRANCH: 'master'
-      REPO_ROOT: ${{ github.workspace }}/src
+      GCC_BPF_INSTALL_DIR: ${{ github.workspace }}/gcc-bpf
+      GCC_BPF_RELEASE_REPO: 'theihor/gcc-bpf'
       KBUILD_OUTPUT: ${{ github.workspace }}/src/kbuild-output
+      REPO_ROOT: ${{ github.workspace }}/src
 
     steps:
 
@@ -45,6 +50,12 @@ jobs:
           dest: ${{ env.REPO_ROOT }}
           rev: ${{ env.BPF_NEXT_BASE_BRANCH }}
 
+      - if: ${{ ! inputs.download_sources }}
+        name: Checkout ${{ github.repository }} to ./src
+        uses: actions/checkout@v4
+        with:
+          path: 'src'
+
       - uses: ./patch-kernel
         with:
           patches-root: '${{ github.workspace }}/ci/diffs'
@@ -65,16 +76,17 @@ jobs:
           arch: ${{ inputs.arch }}
           llvm-version: ${{ inputs.llvm-version }}
 
-      - name: Build GCC BPF compiler
-        uses: ./build-bpf-gcc
-        with:
-          install-dir: ${{ env.BPF_GCC_INSTALL_DIR }}
+      - name: Download GCC BPF compiler
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: .github/scripts/download-gcc-bpf.sh ${{ env.GCC_BPF_RELEASE_REPO }} ${{ env.GCC_BPF_INSTALL_DIR }}
 
       - name: Build selftests/bpf/test_progs-bpf_gcc
         uses: ./build-selftests
         env:
+          BPF_GCC: ${{ env.GCC_BPF_INSTALL_DIR }}
           MAX_MAKE_JOBS: 32
-          BPF_GCC: ${{ env.BPF_GCC_INSTALL_DIR }}
           SELFTESTS_BPF_TARGETS: 'test_progs-bpf_gcc'
         with:
           arch: ${{ inputs.arch }}

.github/workflows/kernel-build-test.yml

@@ -51,6 +51,7 @@ on:
         default: false
 
 jobs:
+
   # Build kernel and selftest
   build:
     uses: ./.github/workflows/kernel-build.yml
@@ -70,7 +71,7 @@ jobs:
       arch: ${{ inputs.arch }}
       toolchain_full: ${{ inputs.toolchain_full }}
       toolchain: ${{ inputs.toolchain }}
-      runs_on: ${{ inputs.runs_on }}
+      runs_on: ${{ inputs.build_runs_on }}
       llvm-version: ${{ inputs.llvm-version }}
       kernel: ${{ inputs.kernel }}
       download_sources: ${{ inputs.download_sources }}
@@ -101,7 +102,8 @@ jobs:
     uses: ./.github/workflows/gcc-bpf.yml
     needs: [build]
     with:
-      runs_on: ${{ inputs.runs_on }}
+      # GCC BPF does not need /dev/kvm, so use the "build" runners
+      runs_on: ${{ inputs.build_runs_on }}
       arch: ${{ inputs.arch }}
       llvm-version: ${{ inputs.llvm-version }}
       toolchain: ${{ inputs.toolchain }}

.github/workflows/kernel-build.yml

@@ -42,22 +42,36 @@ on:
 jobs:
   build:
     name: build for ${{ inputs.arch }} with ${{ inputs.toolchain_full }}${{ inputs.release && '-O2' || '' }}
-    runs-on: ${{ fromJSON(inputs.runs_on) }}
-    timeout-minutes: 100
+    # To run on CodeBuild, runs-on value must correspond to the AWS
+    # CodeBuild project associated with the kernel-patches webhook
+    # However matrix.py passes just a 'codebuild' string
+    runs-on: >-
+      ${{
+          contains(fromJSON(inputs.runs_on), 'codebuild')
+          && format('codebuild-bpf-ci-{0}-{1}', github.run_id, github.run_attempt)
+          || fromJSON(inputs.runs_on)
+      }}
     env:
         ARTIFACTS_ARCHIVE: "vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst"
         BPF_NEXT_BASE_BRANCH: 'master'
         BPF_NEXT_FETCH_DEPTH: 64 # A bit of history is needed to facilitate incremental builds
+        CROSS_COMPILE: ${{ inputs.arch != 'x86_64' && 'true' || '' }}
         # BUILD_SCHED_EXT_SELFTESTS: ${{ inputs.arch == 'x86_64' || inputs.arch == 'aarch64' && 'true' || '' }}
         KBUILD_OUTPUT: ${{ github.workspace }}/kbuild-output
         KERNEL: ${{ inputs.kernel }}
         KERNEL_ROOT: ${{ github.workspace }}
         REPO_PATH: ""
         REPO_ROOT: ${{ github.workspace }}
+        RUNNER_TYPE: ${{ contains(fromJSON(inputs.runs_on), 'codebuild') && 'codebuild' || 'default' }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: ${{ inputs.download_sources && 1 || env.BPF_NEXT_FETCH_DEPTH }}
+
+      - if: ${{ env.RUNNER_TYPE == 'codebuild' }}
+        shell: bash
+        run: .github/scripts/tmpfsify-workspace.sh
+
       - if: ${{ inputs.download_sources }}
         name: Download bpf-next tree
         env:
@@ -66,6 +80,7 @@ jobs:
         with:
           dest: '.kernel'
           rev: ${{ env.BPF_NEXT_BASE_BRANCH }}
+
       - uses: ./prepare-incremental-build
         with:
           repo-root: ${{ inputs.download_sources && '.kernel' || env.REPO_ROOT }}
@@ -99,6 +114,18 @@ jobs:
           llvm-version: ${{ inputs.llvm-version }}
           pahole: master
 
+        # We have to setup qemu+binfmt in order to enable cross-compation of selftests.
+        # During selftests build, freshly built bpftool is executed.
+        # On self-hosted bare-metal hosts binfmt is pre-configured.
+      - if: ${{ env.RUNNER_TYPE == 'codebuild' && env.CROSS_COMPILE }}
+        name: Set up docker
+        uses: docker/setup-docker-action@v4
+      - if: ${{ env.RUNNER_TYPE == 'codebuild' && env.CROSS_COMPILE }}
+        name: Setup binfmt and qemu
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v9.2.0
+
       - name: Build kernel image
         uses: ./build-linux
         with:

.github/workflows/kernel-test.yml

@@ -73,7 +73,7 @@ jobs:
           SELFTESTS_BPF: ${{ github.workspace }}/selftests/bpf
           VMTEST_CONFIGS: ${{ github.workspace }}/ci/vmtest/configs
           TEST_PROGS_TRAFFIC_MONITOR: ${{ inputs.arch == 'x86_64' && 'true' || '' }}
-          TEST_PROGS_WATCHDOG_TIMEOUT: 300
+          TEST_PROGS_WATCHDOG_TIMEOUT: 600
         with:
           arch: ${{ inputs.arch }}
           vmlinuz: '${{ github.workspace }}/vmlinuz'
@@ -83,6 +83,7 @@ jobs:
           # Here we must use kbuild-output local to the repo, because
           # it was extracted from the artifacts.
           kbuild-output: ${{ env.REPO_ROOT }}/kbuild-output
+
       - if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with: