WIP: Various OFED fixes

.github/workflows/package-build-ofed.yml

@@ -1,5 +1,5 @@
 ---
-name: Build OFED packages
+name: Build OFED kernel modules
 on:
   workflow_dispatch:
     inputs:
@@ -23,7 +23,7 @@ env:
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-ofed-packages:
-    name: Build OFED packages
+    name: Build OFED kernel modules
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
@@ -48,6 +48,11 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -86,6 +91,7 @@ jobs:
         id: image_tag
         run: |
           echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name
@@ -191,15 +197,15 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
+          kayobe seed host configure --skip-tags network,docker,docker-registry -e seed_bootstrap_user="cloud-user"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run a distro-sync
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+          kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
@@ -235,12 +241,14 @@ jobs:
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+          TERM: linux
 
       - name: Run OFED upload playbook
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

etc/kayobe/ansible/build-ofed-rocky.yml

@@ -1,5 +1,5 @@
 ---
-- name: Build OFED packages
+- name: Build OFED kernel modules
   become: true
   hosts: ofed-builder
   gather_facts: false
@@ -22,7 +22,6 @@
           - rpm-build
           - automake
           - patch
-          - kernel
           - kernel-devel
           - autoconf
           - pciutils
@@ -42,7 +41,7 @@
 
     - name: Add DOCA host repository package
       ansible.builtin.dnf:
-        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
+        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_{{ stackhpc_pulp_doca_version[:2] }}/doca-host-{{ stackhpc_pulp_doca_version }}-{{ stackhpc_pulp_doca_host_minor_version }}_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
         disable_gpg_check: true
 
     - name: Install DOCA extra packages
@@ -51,23 +50,17 @@
 
     - name: Create build directory
       ansible.builtin.file:
-        path: /home/cloud-user/ofed
+        path: /home/stack/ofed
         state: directory
         mode: 0777
 
     - name: Set build directory
       ansible.builtin.replace:
         path: /opt/mellanox/doca/tools/doca-kernel-support
         regexp: 'TMP_DIR=\$1'
-        replace: 'TMP_DIR=/home/cloud-user/ofed'
+        replace: 'TMP_DIR=/home/stack/ofed'
 
     - name: Build OFED kernel modules
       ansible.builtin.shell:
         cmd: |
-          /opt/mellanox/doca/tools/doca-kernel-support
-
-    - name: Download OFED userspace packages
-      ansible.builtin.dnf:
-        name: doca-ofed-userspace
-        download_only: true
-        download_dir: /home/cloud-user/ofed
+          TERM=linux /opt/mellanox/doca/tools/doca-kernel-support

etc/kayobe/ansible/push-ofed.yml

@@ -1,11 +1,32 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
+  vars:
+    venv: "/opt/kayobe/venvs/kayobe"
   tasks:
     - name: Install python dependencies
       ansible.builtin.pip:
         name: pulp-cli
 
+    - name: Ensure Pulp configuration directory exists
+      ansible.builtin.file:
+        path: /home/stack/.config/pulp/
+        state: directory
+        recurse: true
+
+    - name: Setup Pulp credentials
+      ansible.builtin.blockinfile:
+        path: /home/stack/.config/pulp/cli.toml
+        create: true
+        block: |
+          [cli]
+          base_url = '{{ stackhpc_release_pulp_url }}'
+          verify_ssl = true
+          format = "json"
+          username = '{{ stackhpc_release_pulp_username }}'
+          password = '{{ stackhpc_release_pulp_password }}'
+      no_log: true
+
     - name: Create Pulp repository for OFED
       pulp.squeezer.rpm_repository:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
@@ -17,22 +38,19 @@
 
     - name: Lookup Pulp RPMs on builder
       ansible.builtin.find:
-        paths: "/home/cloud-user/ofed"
+        paths: "/home/stack/ofed"
       register: rpm_dir
 
     - name: Upload OFED RPMs to Pulp
       ansible.builtin.shell:
         cmd: |
-          pulp \
-          --base-url '{{ stackhpc_release_pulp_url }}' \
-          --username '{{ stackhpc_release_pulp_username }}' \
-          --password '{{ stackhpc_release_pulp_password }}' \
+          {{ venv }}/bin/pulp \
           rpm content \
           --type package upload \
           --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
           --file {{ item.path }} \
+          --no-publish \
       with_items: "{{ rpm_dir.files }}"
-      no_log: true
 
     - name: Create Pulp publication for OFED
       pulp.squeezer.rpm_publication:
@@ -48,7 +66,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name + ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}"
+        content_guard: development
+        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ ofed_tag }}"
         state: present

etc/kayobe/dnf.yml

@@ -47,7 +47,26 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
-stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"
+stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_ofed if dnf_install_doca | bool else {}) }}"
+
+# DOCA repositories
+dnf_custom_repos_doca:
+  doca:
+    baseurl: "{{ stackhpc_repo_rocky_doca_url }}"
+    description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }}"
+    enabled: "{{ dnf_enable_doca | bool }}"
+    file: doca
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+  doca-modules:
+    baseurl: "{{ stackhpc_repo_rocky_doca_modules_url }}"
+    description: "DOCA Modules Repo {{ stackhpc_pulp_doca_version }}"
+    enabled: "{{ dnf_enable_doca | bool and dnf_enable_doca_modules | bool }}"
+    file: doca-modules
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
 
 # Custom repositories shared between all RHEL 9 derivatives.
 dnf_custom_repos_el9:
@@ -132,6 +151,12 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}"
 # Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only.
 dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}"
 
+# Whether to enable DOCA repositories. This affects RedHat-based systems only.
+dnf_enable_doca: "{{ dnf_install_doca | bool }}"
+
+# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only.
+dnf_enable_doca_modules: "{{ dnf_install_doca_modules | bool }}"
+
 # URL of EPEL GPG keys.
 dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
 
@@ -152,6 +177,9 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
+# Whether to create a repo file for DOCA. This affects RedHat-based
+# systems only.
+dnf_install_doca: false
 
 ###############################################################################
 # DNF Automatic configuration.

etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm

@@ -0,0 +1,4 @@
+---
+
+seed_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos

@@ -0,0 +1,3 @@
+---
+
+stackhpc_repos_enabled: true

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -95,3 +95,7 @@ stackhpc_docker_registry_password: !vault |
 # Username and password of the overcloud host image repository.
 stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"
+
+# Use image credentials to push OFED RPMs
+stackhpc_release_pulp_username: "{{ stackhpc_image_repository_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_image_repository_password }}"

etc/kayobe/ofed.yml

@@ -1,12 +1,28 @@
 ---
-# DOCA OFED configuration
+# DOCA OFED configuration.
+
+###############################################################################
+# DOCA host version
+stackhpc_pulp_doca_version: 2.8.0
+stackhpc_pulp_doca_minor_version: 204000
 
 # DOCA OFED version
 stackhpc_pulp_doca_ofed_version: 24.07
 
-# DOCA OFED repositories
+###############################################################################
+# Pulp configuration for DOCA OFED
+
+# Whether to sync OFED kernel module packages into the local Pulp service
+stackhpc_pulp_sync_ofed_modules: false
+
+# OFED kernel module repository for Rocky 9
 stackhpc_pulp_repo_doca_ofed_rhel9:
-  name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
-  url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
-  base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+  name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
+  url: "{{ stackhpc_release_pulp_content_url }}/doca_modules/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+  distribution_name: "doca-modules-"
+  base_path: "doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
+  required: "{{ stackhpc_pulp_sync_ofed_modules | bool and stackhpc_pulp_sync_el_9 | bool }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes

etc/kayobe/pulp.yml

@@ -365,6 +365,12 @@ stackhpc_pulp_rpm_repos:
     base_path: "opensearch-dashboards/2.x/yum/"
     required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
+  - name: DOCA Online Repo {{ stackhpc_pulp_doca_host_version }}
+    url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/{{ stackhpc_pulp_repo_doca_version }}"
+    distribution_name: "doca-"
+    base_path: "doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/"
+    required: "{{ stackhpc_pulp_sync_el_9 }}"
+
 # RPM repositories
 stackhpc_pulp_repository_rpm_repos: >-
   {%- set rpm_repos = [] -%}
@@ -380,6 +386,7 @@ stackhpc_pulp_repository_rpm_repos: >-
   {%- endif -%}
   {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(rpm_repo)) -%}
   {%- endfor -%}
+  {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(stackhpc_pulp_repo_doca_ofed_rhel9)) -%}
   {{ rpm_repos }}
 
 # Publication format is a subset of distribution.

etc/kayobe/stackhpc.yml

@@ -52,6 +52,10 @@ stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}"
 ###############################################################################
 # RPMs
 
+# OFED
+stackhpc_repo_ofed_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca_ofed/{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rocky_9_doca_ofed_version }}"
+stackhpc_repo_rocky_9_doca_ofed_version: "{{ stackhpc_repo_distribution }}"
+
 # Grafana
 stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/oss/rpm/{{ stackhpc_repo_grafana_version }}"
 stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}"