Support DOCA OFED

Author: assumptionsandg

etc/kayobe/ansible/build-ofed.yml

@@ -3,15 +3,19 @@
   become: true
   hosts: ofed-builder
   gather_facts: false
-  vars:
-    stackhpc_mlnx_ofed_file_string: MLNX_OFED_LINUX-{{ stackhpc_pulp_mlnx_ofed_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-x86_64
   tasks:
     - name: Extend the home logical volume
       community.general.lvol:
         vg: rootvg
         lv: lv_home
         size: +5G
 
+    - name: Extend the temporary logical volume
+      community.general.lvol:
+        vg: rootvg
+        lv: lv_var_tmp
+        size: +5G
+
     - name: Install package dependencies
       ansible.builtin.dnf:
         name:
@@ -38,38 +42,39 @@
       ansible.builtin.shell:
         cmd: |
           grubby --set-default /boot/$(rpm -qa kernel-devel | sed 's/kernel-devel/vmlinuz/g')
-          echo 'GRUB_DEFAULT=2' >> /etc/default/grub
           grub2-mkconfig -o /boot/grub2/grub.cfg
 
+    - name: Disable noexec in temporary file system
+      ansible.builtin.replace:
+        path: /etc/fstab
+        regexp: '^(exclude=.*)noexec,\*\s*'
+        replace: '\1'
+
     - name: Reboot builder to apply kernel update
       ansible.builtin.reboot:
         reboot_timeout: 600
 
+    - name: Add DOCA host repository
+      ansible.builtin.dnf:
+        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
+
+    - name: Install DOCA extra packages
+      ansible.builtin.dnf:
+        name: doca-extra
+
     - name: Create build directory
       ansible.builtin.file:
         path: /home/cloud-user/ofed
         state: directory
         mode: 0777
 
-    - name: Download MellanoxOFED archive
-      ansible.builtin.get_url:
-        url: https://content.mellanox.com/ofed/MLNX_OFED-{{ stackhpc_pulp_mlnx_ofed_version }}/{{ stackhpc_mlnx_ofed_file_string }}.tgz
-        dest: /home/cloud-user/ofed/ofed-archive.tgz
-
-    - name: Extract MellanoxOFED archive
-      ansible.builtin.unarchive:
-        src: /home/cloud-user/ofed/ofed-archive.tgz
-        dest: /home/cloud-user/ofed
+    - name: Set build directory
+      ansible.builtin.lineinfile:
+        path: /opt/mellanox/doca/tools/doca-kernel-support
+        search_string: 'TMP_DIR=$1'
+        line: '    TMP_DIR=/home/cloud-user/ofed'
 
     - name: Ensure the current kernel is supported
       ansible.builtin.shell:
         cmd: |
-          /home/cloud-user/ofed/{{ stackhpc_mlnx_ofed_file_string }}/mlnx_add_kernel_support.sh \
-          --mlnx_ofed /home/cloud-user/ofed/{{ stackhpc_mlnx_ofed_file_string }} \
-          --tmpdir /home/cloud-user/ofed/ofed-build \
-          --make-tgz -y \
-
-    - name: Extract the new archive
-      ansible.builtin.unarchive:
-        src: /home/cloud-user/ofed/ofed-build/{{ stackhpc_mlnx_ofed_file_string }}-ext.tgz
-        dest: /home/cloud-user/ofed/
+          /opt/mellanox/doca/tools/doca-kernel-support

etc/kayobe/ansible/push-ofed.yml

@@ -2,7 +2,7 @@
 - name: Push OFED packages
   hosts: ofed-builder
   vars:
-    stackhpc_mlnx_ofed_file_string: MLNX_OFED_LINUX-{{ stackhpc_pulp_mlnx_ofed_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-x86_64
+    stackhpc_mlnx_ofed_file_string: MLNX_OFED_LINUX-{{ stackhpc_pulp_doca_ofed_version }}-rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}-x86_64
   tasks:
     - name: Install python dependencies
       ansible.builtin.pip:

etc/kayobe/pulp.yml

@@ -416,14 +416,14 @@ stackhpc_pulp_distribution_rpm_production: >-
 # Mellanox OFED
 
 # Mellanox OFED version
-stackhpc_pulp_mlnx_ofed_version: 24.04-0.7.0.0
+stackhpc_pulp_doca_ofed_version: 24.07
 
 # Mellanox OFED repositories
 stackhpc_pulp_repo_mlnx_ofed_rhel9:
-  name: Mellanox Technologies mlnx_ofed {{ stackhpc_pulp_mlnx_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
-  url: "{{ stackhpc_release_pulp_content_url }}/mlnx_ofed/{{ stackhpc_pulp_mlnx_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_mlnx_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
-  base_path: "mlnx_ofed/{{ stackhpc_pulp_mlnx_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
+  name: Mellanox Technologies mlnx_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
+  url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
+  base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
 
 ###############################################################################
 # Containers