chore(al2023/nvidia): default to 580 driver for 1.34+

Author: mselim00

templates/al2023/provisioners/install-nvidia-driver.sh

@@ -119,7 +119,9 @@ function archive-open-kmods() {
   else
     # Output of `sudo dnf module provides -q kmod-nvidia-open-dkms-570.172.08* | grep Module` is:
     # Module   : nvidia-driver:570-open:20251009011129:f132e61741:x86_64
-    NVIDIA_OPEN_MODULE=$(sudo dnf module provides -q kmod-nvidia-open-dkms-${NVIDIA_DRIVER_FULL_VERSION}* | grep Module | awk -F' : ' '{print $2}')
+    # There is also a latest module stream that may provide the same package version, so we select only
+    # the one with the version in the name
+    NVIDIA_OPEN_MODULE=$(sudo dnf module provides -q kmod-nvidia-open-dkms-${NVIDIA_DRIVER_FULL_VERSION}* | grep Module | awk -F' : ' '{print $2}' | grep "$NVIDIA_DRIVER_MAJOR_VERSION")
     sudo dnf -y module install ${NVIDIA_OPEN_MODULE}
   fi
   dkms status
@@ -232,7 +234,9 @@ function archive-proprietary-kmod() {
   else
     # Output of `sudo dnf module provides -q kmod-nvidia-latest-dkms-570.172.08* | grep Module` is:
     # Module   : nvidia-driver:570-dkms:20251009011129:61f77618b4:x86_64
-    NVIDIA_PROPRIETARY_MODULE=$(sudo dnf module provides -q kmod-nvidia-latest-dkms-${NVIDIA_DRIVER_FULL_VERSION}* | grep Module | awk -F' : ' '{print $2}')
+    # There is also a latest module stream that may provide the same package version, so we select only
+    # the one with the version in the name
+    NVIDIA_PROPRIETARY_MODULE=$(sudo dnf module provides -q kmod-nvidia-latest-dkms-${NVIDIA_DRIVER_FULL_VERSION}* | grep Module | awk -F' : ' '{print $2}' | grep "$NVIDIA_DRIVER_MAJOR_VERSION")
     sudo dnf -y module install ${NVIDIA_PROPRIETARY_MODULE}
   fi
 
@@ -282,14 +286,30 @@ sudo systemctl enable set-nvidia-clocks.service
 ################################################################################
 ### Install other dependencies #################################################
 ################################################################################
-sudo dnf -y install "nvidia-fabric-manager-${NVIDIA_DRIVER_MAJOR_VERSION}.*"
-sudo dnf -y install "nvidia-imex-${NVIDIA_DRIVER_MAJOR_VERSION}.*"
+if [[ "$NVIDIA_DRIVER_MAJOR_VERSION" -lt "580" ]]; then
+  # versions before 580 used to have a dash between fabric and manager
+  sudo dnf -y install "nvidia-fabric-manager-${NVIDIA_DRIVER_FULL_VERSION}"
+else
+  sudo dnf -y install "nvidia-fabricmanager-${NVIDIA_DRIVER_FULL_VERSION}"
+fi
+# versions of nvidia-imex < 580 use nvidia-imex-<major-version>-<full-version>
+sudo dnf -y install "nvidia-imex-${NVIDIA_DRIVER_MAJOR_VERSION}*"
 
 # NVIDIA Container toolkit needs to be locally installed for isolated partitions, also install NVIDIA-Persistenced
 if is-isolated-partition; then
   sudo dnf -y install nvidia-container-toolkit
-  sudo dnf -y install "nvidia-persistenced-${NVIDIA_DRIVER_MAJOR_VERSION}.*"
-  sudo dnf -y install "nvidia-driver-cuda-${NVIDIA_DRIVER_MAJOR_VERSION}.*"
+  sudo dnf -y install "nvidia-persistenced-${NVIDIA_DRIVER_FULL_VERSION}"
+  sudo dnf -y install "nvidia-driver-cuda-${NVIDIA_DRIVER_FULL_VERSION}"
+  # TODO: standardize this across partitions
+  if [[ "$NVIDIA_DRIVER_MAJOR_VERSION" -ge "580" ]]; then
+    sudo dnf -y install \
+      "libnvidia-fbc-${NVIDIA_DRIVER_FULL_VERSION}" \
+      "nvidia-driver-${NVIDIA_DRIVER_FULL_VERSION}" \
+      "nvidia-libXNVCtrl-devel-${NVIDIA_DRIVER_FULL_VERSION}" \
+      "nvidia-settings-${NVIDIA_DRIVER_FULL_VERSION}" \
+      "nvidia-xconfig-${NVIDIA_DRIVER_FULL_VERSION}" \
+      "xorg-x11-nvidia-${NVIDIA_DRIVER_FULL_VERSION}"
+  fi
 else
   sudo dnf -y install nvidia-container-toolkit
 fi

templates/al2023/runtime/gpu/nvidia-kmod-load.sh

@@ -73,6 +73,10 @@ else
   MODULE_NAME="nvidia"
 fi
 
+# TODO: disable dkms autoinstall and write these configurations to /run to ensure even lower
+# priority (more room for user overrides) and expected behavior between reboots
+# dkms install is enabled by default for nvidia modules through the AUTOINSTALL="yes" line
+# in the dkms.conf, which is picked up and ran by dkms.service at boot
 function disable-gsp() {
   echo "options nvidia NVreg_EnableGpuFirmware=0" > /etc/modprobe.d/nvidia-disable-gsp.conf
 }
@@ -84,7 +88,7 @@ case "${INSTANCE_TYPE}" in
   g4dn.* | g5.* | g5g.*)
     echo "Disabling GSP for instance type: ${INSTANCE_TYPE}"
     disable-gsp
-    echo "Using propreitary module for instance type: ${INSTANCE_TYPE}"
+    echo "Using proprietary module for instance type: ${INSTANCE_TYPE}"
     MODULE_NAME="nvidia"
     ;;
 
@@ -93,4 +97,8 @@ case "${INSTANCE_TYPE}" in
     ;;
 esac
 
+# Enable CDMM, only applies for driver versions 580 or later and machines with coherent memory (e.g. GB200)
+# https://nvdam.widen.net/s/gpqp6wmz7s/cuda-whitepaper--cdmm-pdf
+echo "options nvidia NVreg_CoherentGPUMemoryMode=driver" > /etc/modprobe.d/40-eks-nvidia-openrm.conf
+
 kmod-util load "${MODULE_NAME}"