Skip to content

Commit c5db876

Browse files
sjpbsjpb
committed
WIP: move to DOCA
1 parent 4de581c commit c5db876

File tree

3 files changed

+71
-95
lines changed

3 files changed

+71
-95
lines changed

‎ansible/roles/ofed/defaults/main.yml‎

Lines changed: 3 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,3 @@
1-
ofed_version: '23.10-3.2.2.0' # LTS
2-
ofed_download_url: https://content.mellanox.com/ofed/MLNX_OFED-{{ ofed_version }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}.tgz
3-
ofed_distro: rhel # NB: not expected to work on other distros due to installation differences
4-
ofed_distro_version: "{{ ansible_distribution_version }}" # e.g. '8.9'
5-
ofed_distro_major_version: "{{ ansible_distribution_major_version }}" # e.g. '8'
6-
ofed_arch: "{{ ansible_architecture }}"
7-
ofed_tmp_dir: /tmp
8-
ofed_update_firmware: false
9-
ofed_build_packages: # may require additional packages depending on ofed_package_selection
10-
- autoconf
11-
- automake
12-
- gcc
13-
- gcc-gfortran
14-
- kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}
15-
- kernel-rpm-macros
16-
- libtool
17-
- lsof
18-
- patch
19-
- pciutils
20-
- perl
21-
- rpm-build
22-
- tcl
23-
- tk
24-
ofed_build_rl8_packages:
25-
- gdb-headless
26-
- python36
27-
ofed_package_selection: # list of package selection flags for mlnxofedinstall script
28-
- hpc
29-
- with-nfsrdma
1+
doca_version: '2.9.1'
2+
doca_profile: doca-ofed
3+
doca_repo_url: "https://linux.mellanox.com/public/repo/doca/{{ doca_version }}/rhel{{ ansible_distribution_version }}/{{ ansible_architecture }}/"

‎ansible/roles/ofed/tasks/install-kernel-devel.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
- name: Get installed kernels
2+
command: dnf list --installed kernel
3+
register: _ofed_dnf_kernels
4+
changed_when: false
5+
6+
- name: Determine running kernel
7+
command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
8+
register: _ofed_loaded_kernel
9+
changed_when: false
10+
11+
- name: Check current kernel is newest installed
12+
assert:
13+
that: _ofed_kernel_current == _ofed_dnf_kernels_newest
14+
fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
15+
vars:
16+
_ofed_kernel_current: >-
17+
{{ _ofed_loaded_kernel.stdout | regex_replace('\.(?:.(?!\.))+$', '') | regex_replace('\.(?:.(?!\.))+$', '') }}
18+
_ofed_dnf_kernels_newest: >-
19+
{{ _ofed_dnf_kernels.stdout_lines[1:] | map('split') | map(attribute=1) | map('regex_replace', '\.(?:.(?!\.))+$', '') | community.general.version_sort | last }}
20+
# dnf line format e.g. "kernel.x86_64 4.18.0-513.18.1.el8_9 @baseos "
21+
22+
- name: Install matching kernel-devel package
23+
dnf:
24+
name: "kernel-devel-{{ _ofed_loaded_kernel.stdout | trim }}"

‎ansible/roles/ofed/tasks/install.yml‎

Lines changed: 44 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,53 @@
1-
- name: Get installed kernels
2-
command: dnf list --installed kernel
3-
register: _ofed_dnf_kernels
1+
- import_tasks: install-kernel-devel.yml
2+
3+
- name: Install DOCA repo
4+
ansible.builtin.yum_repository:
5+
name: doca
6+
file: doca
7+
description: DOCA Online Repo
8+
baseurl: "{{ doca_repo_url }}"
9+
enabled: true
10+
gpgcheck: false
11+
12+
- name: Install doca-extra package
13+
ansible.builtin.dnf:
14+
name: doca-extra
15+
16+
- name: Build DOCA kernel modules
17+
ansible.builtin.shell:
18+
cmd: /opt/mellanox/doca/tools/doca-kernel-support
19+
register: _doca_kernel_build
20+
21+
22+
- name: Find generated doca-kernel-repo
23+
ansible.builtin.shell: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
24+
register: _doca_kernel_repo # e.g. /tmp/DOCA.WVMchs2QWo/doca-kernel-repo-24.10.1.1.4.0-1.kver.5.14.0.427.31.1.el9.4.x86.64.x86_64.rpm
425
changed_when: false
526

6-
- name: Determine running kernel
7-
command: uname -r # e.g. 4.18.0-513.18.1.el8_9.x86_64
8-
register: _ofed_loaded_kernel
9-
changed_when: false
10-
11-
- name: Check current kernel is newest installed
12-
assert:
13-
that: _ofed_kernel_current == _ofed_dnf_kernels_newest
14-
fail_msg: "Kernel {{ _ofed_loaded_kernel.stdout }} is loaded but newer {{ _ofed_dnf_kernels_newest }} is installed: consider rebooting?"
15-
vars:
16-
_ofed_kernel_current: >-
17-
{{ _ofed_loaded_kernel.stdout | regex_replace('\.(?:.(?!\.))+$', '') | regex_replace('\.(?:.(?!\.))+$', '') }}
18-
_ofed_dnf_kernels_newest: >-
19-
{{ _ofed_dnf_kernels.stdout_lines[1:] | map('split') | map(attribute=1) | map('regex_replace', '\.(?:.(?!\.))+$', '') | community.general.version_sort | last }}
20-
# dnf line format e.g. "kernel.x86_64 4.18.0-513.18.1.el8_9 @baseos "
21-
22-
- name: Enable epel
23-
dnf:
24-
name: epel-release
25-
26-
- name: Check for existing OFED installation
27-
command: ofed_info
28-
changed_when: false
29-
failed_when:
30-
- _ofed_info.rc > 0
31-
- "'No such file or directory' not in _ofed_info.msg"
32-
register: _ofed_info
27+
- name: Create dnf cache
28+
ansible.builtin.command: dnf makecache
3329

34-
- name: Install build prerequisites
35-
dnf:
36-
name: "{{ ofed_build_packages + (ofed_build_rl8_packages if ofed_distro_major_version == '8' else []) }}"
37-
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
38-
# don't want to install a load of prereqs unnecessarily
30+
- name: Install DOCA repository package
31+
ansible.builtin.dnf:
32+
name: "{{ _doca_kernel_repo.stdout }}"
33+
disable_gpg_check: true
3934

40-
- name: Download and unpack Mellanox OFED tarball
41-
ansible.builtin.unarchive:
42-
src: "{{ ofed_download_url }}"
43-
dest: "{{ ofed_tmp_dir }}"
44-
remote_src: yes
45-
become: no
46-
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
35+
- name: Install DOCA packages
36+
ansible.builtin.dnf:
37+
name: "{{ doca_profile }}"
4738

48-
# Below from https://docs.nvidia.com/networking/display/mlnxofedv24010331/user+manual
49-
- name: Run OFED install script
50-
command:
51-
cmd: >
52-
./mlnxofedinstall
53-
--add-kernel-support
54-
{% if not ofed_update_firmware %}--without-fw-update{% endif %}
55-
--force
56-
--skip-repo
57-
{% for pkgsel in ofed_package_selection %}
58-
--{{ pkgsel }}
59-
{% endfor %}
60-
chdir: "{{ ofed_tmp_dir }}/MLNX_OFED_LINUX-{{ ofed_version }}-{{ ofed_distro }}{{ ofed_distro_version }}-{{ ofed_arch }}/"
61-
register: _ofed_install
62-
when: "'MLNX_OFED_LINUX-' + ofed_version not in _ofed_info.stdout"
63-
async: "{{ 45 * 60 }}" # wait for up to 45 minutes
64-
poll: 15 # check every 15 seconds
39+
- name: Cleanup DOCA build directories
40+
ansible.builtin.file:
41+
state: absent
42+
path: "{{ (_doca_kernel_repo.stdout | split('/'))[:2] | join('/') }}"
6543

6644
- name: Update initramfs
67-
command:
68-
cmd: dracut -f
69-
when: '"update your initramfs" in _ofed_install.stdout | default("")'
70-
failed_when: false # always shows errors due to deleted modules for inbox RDMA drivers
45+
ansible.builtin.command:
46+
cmd: dracut -f --tmpdir /var/tmp
47+
environment:
48+
TMPDIR: /var/tmp
49+
register: _doca_dracut
50+
failed_when: _doca_dracut.stderr != '' # appears rc is always 0
7151

7252
- name: Load the new driver
73-
command:
74-
cmd: /etc/init.d/openibd restart
75-
when: '"To load the new driver" in _ofed_install.stdout | default("")'
53+
ansible.builtin.command: /etc/init.d/openibd restart

0 commit comments

Comments
 (0)