Skip to content

Commit 6f92a5b

Browse files
assumptionsandgassumptionsandg
committed
DOCA workflow support
1 parent d397cbd commit 6f92a5b

File tree

13 files changed

+166
-66
lines changed

13 files changed

+166
-66
lines changed

‎.github/workflows/package-build-ofed.yml‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
name: Build OFED packages
2+
name: Build OFED kernel modules
33
on:
44
workflow_dispatch:
55
inputs:
@@ -23,7 +23,7 @@ env:
2323
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
2424
jobs:
2525
overcloud-ofed-packages:
26-
name: Build OFED packages
26+
name: Build OFED kernel modules
2727
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
2828
runs-on: arc-skc-host-image-builder-runner
2929
permissions: {}
@@ -48,6 +48,11 @@ jobs:
4848
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
4949
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
5050
51+
- name: Generate OFED tag
52+
id: ofed_tag
53+
run: |
54+
echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
55+
5156
- name: Clone StackHPC Kayobe repository
5257
uses: actions/checkout@v4
5358
with:
@@ -86,6 +91,7 @@ jobs:
8691
id: image_tag
8792
run: |
8893
echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
94+
working-directory: ${{ github.workspace }}/src/kayobe-config
8995

9096
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
9197
- name: Output image name
@@ -183,23 +189,31 @@ jobs:
183189
run: |
184190
source venvs/kayobe/bin/activate &&
185191
source src/kayobe-config/kayobe-env --environment ci-builder &&
186-
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
192+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml \
193+
-e seed_bootstrap_user="cloud-user" \
194+
-e controller_bootstrap_user="cloud-user" \
187195
env:
188196
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
189197

190198
- name: Configure the seed host (Builder VM)
191199
run: |
192200
source venvs/kayobe/bin/activate &&
193201
source src/kayobe-config/kayobe-env --environment ci-builder &&
194-
kayobe seed host configure --skip-tags network,docker
202+
kayobe seed host configure \
203+
--skip-tags network,docker,docker-registry \
204+
-e seed_bootstrap_user="cloud-user" \
205+
-e controller_bootstrap_user="cloud-user" \
206+
-e dnf_install_doca=true \
207+
-e dnf_enable_doca_modules=false \
208+
-e stackhpc_repos_enabled=true
195209
env:
196210
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
197211

198212
- name: Run a distro-sync
199213
run: |
200214
source venvs/kayobe/bin/activate &&
201215
source src/kayobe-config/kayobe-env --environment ci-builder &&
202-
kayobe seed host command run --become --command "dnf distro-sync --refresh"
216+
kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
203217
env:
204218
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
205219

@@ -240,7 +254,8 @@ jobs:
240254
run: |
241255
source venvs/kayobe/bin/activate &&
242256
source src/kayobe-config/kayobe-env --environment ci-builder &&
243-
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
257+
kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
258+
-e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
244259
env:
245260
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
246261

‎etc/kayobe/ansible/build-ofed-rocky.yml‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
---
2-
- name: Build OFED packages
2+
- name: Build OFED kernel modules
33
become: true
44
hosts: ofed-builder
55
gather_facts: false
@@ -22,7 +22,6 @@
2222
- rpm-build
2323
- automake
2424
- patch
25-
- kernel
2625
- kernel-devel
2726
- autoconf
2827
- pciutils
@@ -37,37 +36,10 @@
3736
- cmake-filesystem
3837
- libnl3-devel
3938
- python3-devel
39+
- doca-extra
4040
state: latest
4141
update_cache: true
4242

43-
- name: Add DOCA host repository package
44-
ansible.builtin.dnf:
45-
name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
46-
disable_gpg_check: true
47-
48-
- name: Install DOCA extra packages
49-
ansible.builtin.dnf:
50-
name: doca-extra
51-
52-
- name: Create build directory
53-
ansible.builtin.file:
54-
path: /home/cloud-user/ofed
55-
state: directory
56-
mode: 0777
57-
58-
- name: Set build directory
59-
ansible.builtin.replace:
60-
path: /opt/mellanox/doca/tools/doca-kernel-support
61-
regexp: 'TMP_DIR=\$1'
62-
replace: 'TMP_DIR=/home/cloud-user/ofed'
63-
6443
- name: Build OFED kernel modules
6544
ansible.builtin.shell:
66-
cmd: |
67-
/opt/mellanox/doca/tools/doca-kernel-support
68-
69-
- name: Download OFED userspace packages
70-
ansible.builtin.dnf:
71-
name: doca-ofed-userspace
72-
download_only: true
73-
download_dir: /home/cloud-user/ofed
45+
cmd: /opt/mellanox/doca/tools/doca-kernel-support

‎etc/kayobe/ansible/push-ofed.yml‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,80 @@
11
---
22
- name: Push OFED packages
33
hosts: ofed-builder
4+
vars:
5+
venv: "/opt/kayobe/venvs/kayobe"
6+
doca_extract_path: "/home/stack/doca"
47
tasks:
8+
- name: Get OFED module repo variables
9+
ansible.builtin.set_fact:
10+
doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='name') | join('') }}"
11+
doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='base_path') | join('') }}"
12+
doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='distribution_name') | join('') }}"
13+
514
- name: Install python dependencies
615
ansible.builtin.pip:
716
name: pulp-cli
817

9-
- name: Create Pulp repository for OFED
18+
- name: Ensure Pulp configuration directory exists
19+
ansible.builtin.file:
20+
path: /home/stack/.config/pulp/
21+
state: directory
22+
recurse: true
23+
24+
- name: Setup Pulp credentials
25+
ansible.builtin.blockinfile:
26+
path: /home/stack/.config/pulp/cli.toml
27+
create: true
28+
block: |
29+
[cli]
30+
base_url = '{{ stackhpc_release_pulp_url }}'
31+
verify_ssl = true
32+
format = "json"
33+
username = '{{ stackhpc_release_pulp_username }}'
34+
password = '{{ stackhpc_release_pulp_password }}'
35+
no_log: true
36+
37+
- name: Find DOCA kernel repo package
38+
ansible.builtin.shell:
39+
cmd: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
40+
register: doca_kernel_repo
41+
changed_when: false
42+
43+
- name: Unpackage kernel repo
44+
ansible.builtin.shell:
45+
cmd: "rpm2cpio {{ doca_kernel_repo.stdout }} | cpio -idmv -D {{ doca_extract_path }}"
46+
47+
- name: Find extracted kernel packages
48+
ansible.builtin.shell:
49+
cmd: 'find {{ doca_extract_path }}/usr/share/doca-host-*/Modules/$(uname -r)/*.rpm'
50+
register: doca_kernel_packages
51+
52+
- name: Create Pulp repository for DOCA kernel modules
1053
pulp.squeezer.rpm_repository:
1154
pulp_url: "{{ stackhpc_release_pulp_url }}"
1255
username: "{{ stackhpc_release_pulp_username }}"
1356
password: "{{ stackhpc_release_pulp_password }}"
14-
name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
57+
name: "{{ doca_modules_repo_name }}"
1558
state: present
1659
retries: "{{ pulp_timeout_retries | default(3) }}"
1760

18-
- name: Lookup Pulp RPMs on builder
19-
ansible.builtin.find:
20-
paths: "/home/cloud-user/ofed"
21-
register: rpm_dir
22-
2361
- name: Upload OFED RPMs to Pulp
2462
ansible.builtin.shell:
2563
cmd: |
26-
pulp \
27-
--base-url '{{ stackhpc_release_pulp_url }}' \
28-
--username '{{ stackhpc_release_pulp_username }}' \
29-
--password '{{ stackhpc_release_pulp_password }}' \
64+
{{ venv }}/bin/pulp \
3065
rpm content \
3166
--type package upload \
32-
--repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
33-
--file {{ item.path }} \
34-
with_items: "{{ rpm_dir.files }}"
35-
no_log: true
67+
--repository '{{ doca_modules_repo_name }}' \
68+
--file {{ item }} \
69+
--no-publish \
70+
with_items: "{{ doca_kernel_packages.stdout_lines }}"
3671

3772
- name: Create Pulp publication for OFED
3873
pulp.squeezer.rpm_publication:
3974
pulp_url: "{{ stackhpc_release_pulp_url }}"
4075
username: "{{ stackhpc_release_pulp_username }}"
4176
password: "{{ stackhpc_release_pulp_password }}"
42-
repository: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
77+
repository: "{{ doca_modules_repo_name }}"
4378
state: present
4479
register: publication
4580

@@ -48,7 +83,8 @@
4883
pulp_url: "{{ stackhpc_release_pulp_url }}"
4984
username: "{{ stackhpc_release_pulp_username }}"
5085
password: "{{ stackhpc_release_pulp_password }}"
51-
name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}"
86+
name: "{{ doca_modules_repo_distribution_name + ofed_tag }}"
5287
publication: "{{ publication.publication.pulp_href }}"
53-
base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}"
88+
content_guard: development
89+
base_path: "{{ doca_modules_repo_base_path + ofed_tag }}"
5490
state: present

‎etc/kayobe/dnf.yml‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,28 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
4747
# To use these repos, set stackhpc_repos_enabled to true.
4848
# This is done by default for hosts in the overcloud group via a group_vars
4949
# file.
50-
stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"
50+
stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_doca if dnf_install_doca | bool else {}) }}"
51+
52+
# DOCA repositories
53+
dnf_custom_repos_doca:
54+
doca:
55+
baseurl: "{{ stackhpc_repo_rhel9_doca_url }}"
56+
description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
57+
enabled: "{{ dnf_enable_doca | bool | default(false) }}"
58+
priority: -1
59+
file: doca
60+
gpgcheck: no
61+
username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
62+
password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
63+
doca-modules:
64+
baseurl: "{{ stackhpc_repo_rhel9_doca_modules_url }}"
65+
description: "OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_version }} - RHEL $releasever"
66+
enabled: "{{ dnf_enable_doca_modules | bool | default(false) }}"
67+
priority: -2
68+
file: doca
69+
gpgcheck: no
70+
username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
71+
password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
5172

5273
# Custom repositories shared between all RHEL 9 derivatives.
5374
dnf_custom_repos_el9:
@@ -132,6 +153,12 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}"
132153
# Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only.
133154
dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}"
134155

156+
# Whether to enable DOCA repositories. This affects RedHat-based systems only.
157+
dnf_enable_doca: "{{ dnf_install_doca | bool }}"
158+
159+
# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only.
160+
dnf_enable_doca_modules: "{{ dnf_install_doca | bool }}"
161+
135162
# URL of EPEL GPG keys.
136163
dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
137164

@@ -152,6 +179,9 @@ dnf_enable_docker: true
152179
#URL of docker repo GPG key
153180
dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
154181

182+
# Whether to create a repo file for DOCA. This affects RedHat-based
183+
# systems only. Default value is 'false'.
184+
#dnf_install_doca:
155185

156186
###############################################################################
157187
# DNF Automatic configuration.

‎etc/kayobe/environments/ci-aio/stackhpc-ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_vers
5050
stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
5151
stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
5252
stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
53+
stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
54+
stackhpc_repo_rhel9_doca_modules_version: "{{ stackhpc_pulp_repo_rhel9_doca_modules_version }}"
5355

5456
# Rocky-and-CI-specific Pulp urls
5557
stackhpc_include_os_minor_version_in_repo_url: true

‎etc/kayobe/environments/ci-builder/inventory/group_vars/seed/lvm.yml‎ renamed to ‎etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
---
2+
23
# List of extra LVs to include in the rootvg VG.
34
stackhpc_lvm_group_rootvg_lvs_extra:
45
- "{{ stackhpc_lvm_lv_docker }}"
@@ -16,7 +17,7 @@ stackhpc_lvm_lv_root_size: 2g
1617
stackhpc_lvm_lv_tmp_size: 2g
1718

1819
# StackHPC LVM lv_var LV size.
19-
stackhpc_lvm_lv_var_size: 2g
20+
stackhpc_lvm_lv_var_size: 3g
2021

2122
# StackHPC LVM lv_var_tmp LV size.
2223
stackhpc_lvm_lv_var_tmp_size: 2g

‎etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
---
2+
3+
stackhpc_repos_enabled: true

‎etc/kayobe/environments/ci-builder/inventory/hosts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
[ofed-builder:children]
44
seed
5+
controllers
56

67
# This is neccesary to run `overcloud deployment image build`
78
[controllers]

‎etc/kayobe/environments/ci-builder/stackhpc-ci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_vers
7575
stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
7676
stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
7777
stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
78+
stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
7879

7980
# Rocky-and-CI-specific Pulp urls
8081
stackhpc_include_os_minor_version_in_repo_url: true
@@ -107,3 +108,9 @@ ipa_build_dib_env_extra:
107108
# builder being a member of the 'overcloud' group for IPA builds.
108109
stackhpc_repos_enabled: false
109110
enable_docker_repo: true
111+
112+
# Enable LVM for OFED builds
113+
seed_lvm_groups:
114+
- "{{ stackhpc_lvm_group_rootvg if dnf_install_doca | bool }}"
115+
controller_lvm_groups:
116+
- "{{ stackhpc_lvm_group_rootvg if dnf_install_doca | bool }}"

‎etc/kayobe/ofed.yml‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,21 @@
11
---
2-
# DOCA OFED configuration
2+
# DOCA OFED configuration.
33

4-
# DOCA OFED version
5-
stackhpc_pulp_doca_ofed_version: 24.07
4+
###############################################################################
5+
# DOCA host version
6+
stackhpc_pulp_doca_version: 2.9.1
67

7-
# DOCA OFED repositories
8-
stackhpc_pulp_repo_doca_ofed_rhel9:
9-
name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
10-
url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
11-
distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
12-
base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
8+
###############################################################################
9+
# Pulp configuration for DOCA OFED
10+
11+
# Whether to sync OFED kernel module packages into the local Pulp service
12+
stackhpc_pulp_sync_ofed_modules: false
13+
14+
# DOCA Snapshot versions. The defaults use the appropriate version from
15+
# pulp-repo-versions.yml
16+
stackhpc_pulp_repo_rhel9_doca_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_version') }}"
17+
stackhpc_pulp_repo_rhel9_doca_modules_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rhel_9_{{ stackhpc_pulp_repo_rocky_9_minor_version }}_doca_modules_version') }}"
18+
19+
###############################################################################
20+
# Dummy variable to allow Ansible to accept this file.
21+
workaround_ansible_issue_8743: yes

0 commit comments

Comments
 (0)