24.11.0 release fix

Author: angudadev

playbooks/nvidia-driver.yaml

@@ -149,28 +149,14 @@
       become: true
       get_url:
         url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-        dest: /tmp//cuda-{{ ansible_distribution | lower }}.repo
+        dest: /etc/yum.repos.d/cuda-{{ ansible_distribution | lower }}.repo
 
     - name: Download NVIDIA CUDA keyring package for RHEL
       when: ansible_distribution == 'RedHat' and  ansible_architecture == 'aarch64'
       become: true
       get_url:
         url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/sbsa/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-        dest: /tmp//cuda-{{ ansible_distribution | lower }}.repo
-
-    - name: Install NVIDIA CUDA keyring package for RHEL
-      when: ansible_distribution == 'RedHat'
-      become: true
-      block:
-        - name: Install CUDA keyring package
-          yum:
-            name: /tmp/cuda-{{ ansible_distribution | lower }}.repo
-            state: present
-
-        - name: Clean up the downloaded package
-          file:
-            path: /tmp//cuda-{{ ansible_distribution | lower }}.repo
-            state: absent
+        dest: /etc/yum.repos.d/cuda-{{ ansible_distribution | lower }}.repo
 
     - name:  ensure we have kernel-headers installed for the current kernel on RHEL
       when:  "cns_version >= 10.0 and ansible_distribution == 'RedHat'"
@@ -251,14 +237,53 @@
     - set_fact:
         driver_version: "{{ dversion.stdout }}"
 
+    - name: check if NVSwitch/NVlink
+      shell: lspci | grep -i nvidia | egrep -i 'bridge|nvlink|nvswitch'
+      register: nvlink
+      failed_when: false
+
+    - name: check nvlink status with NVIDIA SMI
+      shell: nvidia-smi nvlink -s -i 0 | tail -1f | awk '{print $NF}' | tr -dd '\n'
+      register: nvlink_status
+      failed_when: false
+
+    - name: check dgx
+      ignore_errors: true
+      stat:
+        path: /etc/dgx-release
+      register: dgx
+
+    - name: check l4t
+      ignore_errors: true
+      stat:
+        path: /etc/l4t-release
+      register: l4t
+
     - name: Install NVIDIA Fabric Manager on Ubuntu
       become: true
-      when: ansible_distribution == 'Ubuntu'
+      when: ansible_distribution == 'Ubuntu' and dgx.stat.exists == True or ansible_distribution == 'Ubuntu' and l4t.stat.exists == True or ansible_distribution == 'Ubuntu' and nvlink_status.stdout != 'inActive'
       ignore_errors: true
-      shell: "apt update; apt install nvidia-fabricmanager-{{ driver_version }} -y; sudo systemctl --now enable nvidia-fabricmanager; sudo systemctl daemon-reload"
+      apt:
+        name: "nvidia-fabricmanager-{{ driver_version }}={{ gpu_driver_version}}-1"
+        state: present
+        update_cache: true
+        force: yes
 
-    - name: Install NVIDIA Fabric Manager on RHEL
+    - name: Install NVIDIA Fabric Manager for NVSwitch on RHEL
       become: true
-      when: ansible_distribution == 'RedHat'
+      when: "ansible_distribution == 'RedHat' and dgx.stat.exists == True or ansible_distribution == 'RedHat' and l4t.stat.exists == True or ansible_distribution == 'RedHat' and and nvlink_status.stdout != 'inActive'"
       ignore_errors: true
-      shell: "yum update -y; yum install nvidia-fabricmanager-{{ driver_version }} -y; sudo systemctl --now enable nvidia-fabricmanager; sudo systemctl daemon-reload"
+      yum:
+        name: "nvidia-fabric-manager-{{ gpu_driver_version}}-1"
+        state: present
+        update_cache: true
+
+    - name: Enable and restart NVIDIA Fabric manager
+      when: dgx.stat.exists == True or l4t.stat.exists == True or nvlink_status.stdout != 'inActive'
+      ignore_errors: true
+      become: true
+      systemd_service:
+        name: nvidia-fabricmanager
+        enabled: true
+        state: started
+        daemon_reload: true

playbooks/operators-install.yaml

@@ -272,6 +272,10 @@
      when: "confidential_computing == false and enable_gpu_operator == true and use_open_kernel_module == true and enable_mig == true and  enable_rdma == true and enable_vgpu == false and enable_gds == false and enable_secure_boot == false and gpu_operator.rc == 1 and network_operator_valid.rc == 1 and cns_nvidia_driver == false and ngc_registry_password == ''"
      shell:  helm install --version {{ gpu_operator_version }} --values {{ ansible_user_dir }}/values.yaml --create-namespace --namespace nvidia-gpu-operator --devel '{{ gpu_operator_helm_chart }}' --set driver.rdma.enabled=true,driver.rdma.useHostMofed=true,mig.strategy='{{ mig_strategy }}',driver.version='{{ gpu_driver_version }}',driver.repository='{{ gpu_operator_driver_registry }}',driver.useOpenKernelModules=true --wait --generate-name
 
+   - name: Installing the GDS with Open RM GPU Operator on NVIDIA Cloud Native Stack
+     when: "confidential_computing == false and enable_gpu_operator == true and use_open_kernel_module == true and enable_mig == false and  enable_rdma == false and enable_vgpu == false and enable_gds == true and enable_secure_boot == false and gpu_operator.rc == 1 and network_operator_valid.rc == 1 and cns_nvidia_driver == false and ngc_registry_password == ''"
+     shell:  helm install --version {{ gpu_operator_version }} --values {{ ansible_user_dir }}/values.yaml --create-namespace --namespace nvidia-gpu-operator --devel '{{ gpu_operator_helm_chart }}' --set gds.enabled=true,driver.version='{{ gpu_driver_version }}',driver.repository='{{ gpu_operator_driver_registry }}',driver.useOpenKernelModules=true --wait --generate-name
+
    - name: Installing the GPU Operator with RDMA and Host MOFED on NVIDIA Cloud Native Stack
      when: "confidential_computing == false and enable_gpu_operator == true and use_open_kernel_module == false and enable_mig == false and  enable_rdma == true and enable_vgpu == false and enable_gds == false and enable_secure_boot == false and gpu_operator.rc == 1 and network_operator_valid.rc == 1 and cns_nvidia_driver == false and ngc_registry_password == ''"
      shell:  helm install --version {{ gpu_operator_version }} --values {{ ansible_user_dir }}/values.yaml --create-namespace --namespace nvidia-gpu-operator --devel '{{ gpu_operator_helm_chart }}' --set driver.rdma.enabled=true,driver.rdma.useHostMofed=true,driver.version='{{ gpu_driver_version }}',driver.repository='{{ gpu_operator_driver_registry }}' --wait --generate-name

playbooks/prerequisites.yaml

@@ -87,6 +87,51 @@
         name: ['containers-common','net-tools', 'libseccomp', 'curl', 'ca-certificates', 'kubelet-{{ k8s_version }}', 'kubeadm-{{ k8s_version }}', 'kubectl-{{ k8s_version }}']
         state: present
 
+    - name: check update GCC for build essentials
+      when: "ansible_distribution == 'Ubuntu'"
+      become: true
+      block:
+        - name: capture gcc expected version
+          shell: cat /proc/version | tr -d '(' | tr -d ')' | awk '{print $8}' | cut -d '.' -f 1
+          register: gcc_expected_version
+
+        - name: install expected gcc version
+          apt:
+            name: "gcc-{{ gcc_expected_version.stdout }}"
+            state: present
+
+        - name: install expected g++ version
+          apt:
+            name: "g++-{{ gcc_expected_version.stdout }}"
+            state: present
+
+        - name: capture gcc binary path
+          shell: "which gcc"
+          register: gcc_bin_path
+
+        - name: capture g++ binary path
+          shell: "which g++"
+          register: g_plus_plus_bin_path
+
+        - name: capture gcc expected version binary path
+          shell: "which gcc-{{ gcc_expected_version.stdout }}"
+          register: gcc_expected_version_bin_path
+
+        - name: capture g++ expected version binary path
+          shell: "which g++-{{ gcc_expected_version.stdout }}"
+          register: g_plus_plus_expected_version_bin_path
+
+        - name: setup expected gcc as primary gcc
+          community.general.alternatives:
+            name: gcc
+            link: "{{ gcc_bin_path.stdout }}"
+            path: "{{ gcc_expected_version_bin_path.stdout }}"
+            subcommands:
+            - name: g++
+              link: "{{ g_plus_plus_bin_path.stdout }}"
+              path: "{{ g_plus_plus_expected_version_bin_path.stdout }}"
+            state: auto
+
     - name: Hold the installed Packages
       become: true
       when: "ansible_distribution == 'Ubuntu'"