Merge pull request #2798 from flatcar/danzatt/nvidia-drivers-signing

Add prebuilt NVIDIA drivers in a sysext

- Add capability to specify per-sysext USE flags and compile different versions of upstream portage nvidia-drivers (including open and non-open variants).
- Allow architecture-specific OS-dependent sysexts
- Pull `nvidia-drivers` from portage and build sysexts from the package

Related PRs:
NVIDIA tests using sysext: [mantle #598](flatcar/mantle#598)
NVIDIA runtime modifications to remove `nvidia-smi` symlink: [sysext-bakery #153](flatcar/sysext-bakery#153)

Author: danzatt

.github/workflows/portage-stable-packages-list

@@ -51,6 +51,7 @@ acct-user/named
 acct-user/netperf
 acct-user/nobody
 acct-user/ntp
+acct-user/nvpd
 acct-user/pcap
 acct-user/pcscd
 acct-user/polkitd

build_image

@@ -33,7 +33,7 @@ DEFINE_string base_pkg "coreos-base/coreos" \
   "The base portage package to base the build off of (only applies to prod images)"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
   "The base portage package to base the build off of (only applies to dev containers)"
-DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
+DEFINE_string base_sysexts "containerd-flatcar|app-containers/containerd,docker-flatcar|app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
   "Comma-separated list of name:package[&package[&package]] - build 'package' (a single package or a list of packages separated by '&') into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"

build_library/extra_sysexts.sh

@@ -1,5 +1,27 @@
 EXTRA_SYSEXTS=(
-  zfs:sys-fs/zfs
-  podman:app-containers/podman,net-misc/passt
-  python:dev-lang/python,dev-python/pip
+  "zfs|sys-fs/zfs"
+  "podman|app-containers/podman,net-misc/passt"
+  "python|dev-lang/python,dev-python/pip"
+  "nvidia-drivers-535|x11-drivers/nvidia-drivers:0/535|-kernel-open persistenced|amd64"
+  "nvidia-drivers-535-open|x11-drivers/nvidia-drivers:0/535|kernel-open persistenced|amd64"
+  "nvidia-drivers-550|x11-drivers/nvidia-drivers:0/550|-kernel-open persistenced|amd64"
+  "nvidia-drivers-550-open|x11-drivers/nvidia-drivers:0/550|kernel-open persistenced|amd64"
+  "nvidia-drivers-570|x11-drivers/nvidia-drivers:0/570|-kernel-open persistenced|amd64"
+  "nvidia-drivers-570-open|x11-drivers/nvidia-drivers:0/570|kernel-open persistenced|amd64"
 )
+
+_get_unversioned_sysext_packages_unsorted() {
+  for sysext in "${EXTRA_SYSEXTS[@]}"; do
+    IFS="|" read -r _ PACKAGE_ATOMS _ <<< "$sysext"
+
+    IFS=,
+    for atom in $PACKAGE_ATOMS; do
+       qatom "$atom" -F "%{CATEGORY}/%{PN}"
+    done
+    unset IFS
+  done
+}
+
+get_unversioned_sysext_packages() {
+  _get_unversioned_sysext_packages_unsorted | sort | uniq
+}

build_library/prod_image_util.sh

@@ -213,18 +213,36 @@ create_prod_sysexts() {
   local image_name="$1"
   local image_sysext_base="${image_name%.bin}_sysext.squashfs"
   for sysext in "${EXTRA_SYSEXTS[@]}"; do
-    local name="flatcar-${sysext%:*}"
-    local pkgs="${sysext#*:}"
+    local name pkgs useflags arches
+    IFS="|" read -r name pkgs useflags arches <<< "$sysext"
+    name="flatcar-$name"
     local pkg_array=(${pkgs//,/ })
+    local arch_array=(${arches//,/ })
+    local useflags_array=(${useflags//,/ })
+
     local mangle_script="${BUILD_LIBRARY_DIR}/sysext_mangle_${name}"
     if [[ ! -x "${mangle_script}" ]]; then
       mangle_script=
     fi
+
+    if [[ -n "$arches" ]]; then
+      should_skip=1
+      for arch in "${arch_array[@]}"; do
+        if [[ $arch == "$ARCH" ]]; then
+          should_skip=0
+        fi
+      done
+      if [[ $should_skip -eq 1 ]]; then
+        continue
+      fi
+    fi
+
     sudo rm -f "${BUILD_DIR}/${name}.raw" \
 	"${BUILD_DIR}/flatcar-test-update-${name}.gz" \
 	"${BUILD_DIR}/${name}_*"
-    sudo "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
-        --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
+    # we use -E to pass the USE flags, but also MODULES_SIGN variables
+    USE="${useflags_array[*]}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
+	--squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
 	--image_builddir="${BUILD_DIR}" \
 	${mangle_script:+--manglefs_script=${mangle_script}} \
 	"${name}" "${pkg_array[@]}"

build_library/sysext_prod_builder

@@ -113,8 +113,8 @@ prev_pkginfo=""
 sysext_lowerdirs="${sysext_mountdir}/rootfs-lower"
 for sysext in ${sysexts_list//,/ }; do
   # format is "<name>:<group>/<package>"
-  name="${sysext%:*}"
-  grp_pkg="${sysext#*:}"
+  name="${sysext%|*}"
+  grp_pkg="${sysext#*|}"
   create_prod_sysext "${BOARD}" \
                      "${sysext_output_dir}" \
                      "${sysext_workdir}" \

build_packages

@@ -117,6 +117,7 @@ fi
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
 
 # Setup all the emerge command/flags.
 EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
@@ -285,6 +286,48 @@ export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"
 info "Merging board packages now"
 sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"
 
+info "Merging sysext packages now"
+for sysext in "${EXTRA_SYSEXTS[@]}"; do
+  IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext"
+
+  arch_array=("${ARCHES//,/ }")
+  if [[ -n $ARCHES ]]; then
+    should_skip=1
+    for arch in "${arch_array[@]}"; do
+      if [[ $arch == "$ARCH" ]]; then
+        should_skip=0
+      fi
+    done
+    if [[ $should_skip -eq 1 ]]; then
+      continue
+    fi
+  fi
+
+
+  info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS"
+  IFS=,
+  for package in $PACKAGE_ATOMS; do
+     # --buildpkgonly does not install dependencies, so we install them
+     # separately before building the binary package
+     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
+       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
+       "${EMERGE_FLAGS[@]}" \
+       --quiet \
+       --onlydeps \
+       --binpkg-respect-use=y \
+       "${package}"
+
+     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
+       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
+       "${EMERGE_FLAGS[@]}" \
+       --quiet \
+       --buildpkgonly \
+       --binpkg-respect-use=y \
+       "${package}"
+  done
+  unset IFS
+done
+
 info "Removing obsolete packages"
 # The return value of emerge is not clearly reliable. It may fail with
 # an output like following:
@@ -319,7 +362,16 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then
   fi
 fi
 
-eclean-$BOARD -d packages
+exclusions_file=$(mktemp)
+if [ ! -f "$exclusions_file" ]; then
+    die_notrace "Couldn't create temporary exclusions file $exclusions_file for eclean"
+fi
+get_unversioned_sysext_packages > "$exclusions_file"
+eclean-"$BOARD" -d --exclude-file="$exclusions_file" packages
+rm -f "$exclusions_file"
+# run eclean again, this time without the --deep option, to clean old versions
+# of sysext packages (those, for which .ebuild file no  longer exists)
+eclean-"$BOARD" packages
 
 info "Checking build root"
 test_image_content "${BOARD_ROOT}"

build_sysext

@@ -221,11 +221,12 @@ info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_
 
 for package; do
   echo "Installing package into sysext image: $package"
-  FEATURES="-ebuild-locks" emerge \
+  FEATURES="-ebuild-locks binpkg-multi-instance" emerge \
     --root="${BUILD_DIR}/install-root" \
     --config-root="/build/${FLAGS_board}"  \
     --sysroot="/build/${FLAGS_board}"  \
     --usepkgonly \
+    --binpkg-respect-use=y \
     --getbinpkg \
     --verbose \
     --jobs=${NUM_JOBS} \

changelog/changes/2025-05-13-nvidia-sysext.md

@@ -0,0 +1,2 @@
+- Compile OS-dependent NVIDIA kernel module sysexts signed for secure boot. ([scripts#2798](https://github.com/flatcar/scripts/pull/2798/))
+- Allow per-sysext USE flags and architecture-specific sysexts. ([scripts#2798](https://github.com/flatcar/scripts/pull/2798/))

changelog/changes/2025-05-13-oot-module-signing.md

@@ -0,0 +1 @@
+- Sign out-of-tree kernel modules using the ephemeral signing key so that ZFS and NVIDIA sysexts can work with secure boot. ([scripts#2636](https://github.com/flatcar/scripts/pull/2636/))

ci-automation/base_sysexts.sh

@@ -6,6 +6,6 @@ if [[ ${1:-} = 'local' ]]; then
 fi
 
 ciabs_base_sysexts=(
-    'containerd-flatcar:app-containers/containerd'
-    'docker-flatcar:app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx'
+    'containerd-flatcar|app-containers/containerd'
+    'docker-flatcar|app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx'
 )