add doca role run by fatimage

Author: sjpb

ansible/.gitignore

@@ -66,3 +66,5 @@ roles/*
 !roles/lustre/**
 !roles/dnf_repos/
 !roles/dnf_repos/**
+!roles/doca/
+!roles/doca/**

ansible/fatimage.yml

@@ -199,6 +199,14 @@
         name: cloudalchemy.grafana
         tasks_from: install.yml
 
+- hosts: doca
+  become: yes
+  gather_facts: yes
+  tasks:
+    - name: Install Mellanox DOCA
+      include_role:
+        name: doca
+
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"

ansible/roles/doca/defaults/main.yml

@@ -0,0 +1,3 @@
+doca_version: '2.9.1' # 2.9 is LTS, last to support ConnectX-4, 3 years for bug fixes and CVE updates
+doca_profile: doca-ofed
+doca_repo_url: "https://linux.mellanox.com/public/repo/doca/{{ doca_version }}/rhel{{ ansible_distribution_version }}/{{ ansible_architecture }}/"

ansible/roles/doca/tasks/install-kernel-devel.yml

@@ -0,0 +1,24 @@
+- name: Get installed kernels
+  command: dnf list --installed kernel
+  register: _ofed_dnf_kernels
+  changed_when: false
+
+- name: Determine running kernel
+  command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
+  register: _ofed_loaded_kernel
+  changed_when: false
+
+- name: Check current kernel is newest installed
+  assert:
+    that: _ofed_kernel_current == _ofed_dnf_kernels_newest
+    fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
+  vars:
+    _ofed_kernel_current: >-
+      {{ _ofed_loaded_kernel.stdout | regex_replace('\.(?:.(?!\.))+$', '') | regex_replace('\.(?:.(?!\.))+$', '') }}
+    _ofed_dnf_kernels_newest: >-
+      {{ _ofed_dnf_kernels.stdout_lines[1:] | map('split') | map(attribute=1) | map('regex_replace', '\.(?:.(?!\.))+$', '') | community.general.version_sort | last }}
+    # dnf line format e.g. "kernel.x86_64  4.18.0-513.18.1.el8_9   @baseos  "
+
+- name: Install matching kernel-devel package
+  dnf:
+    name: "kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}"

ansible/roles/doca/tasks/install.yml

@@ -0,0 +1,53 @@
+- import_tasks: install-kernel-devel.yml
+
+- name: Install DOCA repo
+  ansible.builtin.yum_repository:
+    name: doca
+    file: doca
+    description: DOCA Online Repo
+    baseurl: "{{ doca_repo_url }}"
+    enabled: true
+    gpgcheck: false
+
+- name: Install doca-extra package
+  ansible.builtin.dnf:
+    name: doca-extra
+
+- name: Build DOCA kernel modules
+  ansible.builtin.shell:
+    cmd: /opt/mellanox/doca/tools/doca-kernel-support
+  register: _doca_kernel_build
+
+
+- name: Find generated doca-kernel-repo
+  ansible.builtin.shell: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
+  register: _doca_kernel_repo # e.g. /tmp/DOCA.WVMchs2QWo/doca-kernel-repo-24.10.1.1.4.0-1.kver.5.14.0.427.31.1.el9.4.x86.64.x86_64.rpm
+  changed_when: false
+
+- name: Create dnf cache
+  ansible.builtin.command: dnf makecache
+
+- name: Install DOCA repository package
+  ansible.builtin.dnf:
+    name: "{{ _doca_kernel_repo.stdout }}"
+    disable_gpg_check: true
+
+- name: Install DOCA packages
+  ansible.builtin.dnf:
+    name: "{{ doca_profile }}"
+
+- name: Cleanup DOCA build directories
+  ansible.builtin.file:
+    state: absent
+    path: "{{ (_doca_kernel_repo.stdout | split('/'))[:2] | join('/') }}"
+
+- name: Update initramfs
+  ansible.builtin.command:
+    cmd: dracut -f --tmpdir /var/tmp
+  environment:
+     TMPDIR: /var/tmp
+  register: _doca_dracut
+  failed_when: _doca_dracut.stderr != '' # appears rc is always 0
+
+- name: Load the new driver
+  ansible.builtin.command: /etc/init.d/openibd restart