DOCA workflow support

.github/workflows/package-build-ofed.yml

@@ -1,5 +1,5 @@
 ---
-name: Build OFED packages
+name: Build OFED kernel modules
 on:
   workflow_dispatch:
     inputs:
@@ -23,7 +23,7 @@ env:
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-ofed-packages:
-    name: Build OFED packages
+    name: Build OFED kernel modules
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
@@ -48,6 +48,11 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -86,6 +91,7 @@ jobs:
         id: image_tag
         run: |
           echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name
@@ -183,23 +189,31 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/growroot.yml \
+          -e seed_bootstrap_user="cloud-user" \
+          -e controller_bootstrap_user="cloud-user" \
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Configure the seed host (Builder VM)
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
+          kayobe seed host configure \
+          --skip-tags network,docker,docker-registry \
+          -e seed_bootstrap_user="cloud-user" \
+          -e controller_bootstrap_user="cloud-user" \
+          -e dnf_install_doca=true \
+          -e dnf_enable_doca_modules=false \
+          -e stackhpc_repos_enabled=true
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Run a distro-sync
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+          kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
@@ -240,7 +254,8 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

doc/source/contributor/ofed.rst

@@ -4,19 +4,17 @@ OFED
 
 Warning: Experimental workflow subject to change
 
-This section documents the workflow for building OFED packages for Release train integration.
-
-The workflow builds the OFED kernel modules against the latest available kernel in Release train
-(as configured in SKC) and compiles them into RPM packages to be uploaded to Ark. Addtionally,
-this workflow downloads the userspace OFED packages from the Nvidia repository and uploads these
-to Ark.
+The Nvidia DOCA framework is distributed as part of StackHPC Release Train for OFED driver support,
+this repository is synced into Ark as part of the Release Train worfkflows, however to ensure
+compatibility with Release Train packages, we are required to build OFED modules with support for
+the latest Release Train kernel.
 
 Workflow
 ========
 
 The workflow uses workflow_dispatch to manually request an OFED build, which will deploy a builder
 VM, apply kayobe config to the builder, upgrade the kernel, reboot, then run two Ansible playbooks
-for building and uploading OFED to Ark.
+for building and uploading OFED modules to Ark.
 
 Pre-requisites
 --------------
@@ -25,31 +23,54 @@ Before building OFED packages, the workflow will ensure that:
 
 * A full distro-sync has taken place, ensuring the kernel is upgraded.
 
-* The bootloader has been configured to use the latest kernel
+* The bootloader has been configured to use the latest kernel (reset-bls-entries.yml)
 
 * noexec is disabled in the temporary logical volume.
 
 build-ofed
 ----------
 
-Currently we only support building Rocky Linux 9 OFED packages.
-
-In order to setup OFED, we're required to build kernel modules for the OFED drivers as
-the kernels we provide in release train are unsupported by OFED. To accomplish this we
-will need to use the doca-kernel-support from the doca-extra repository.
+Currently we only support building Rocky Linux 9 OFED kerenl module packages.
 
-We will need to instll dependencies in order to build the OFED kernel modules, and these
-are installed at the beginning of the build playbook. We also install base and appstream
-dependencies of userspace OFED packages here, this is intended to stop these dependencies
-being pulled in later when we download the OFED packages from the doca-host repository.
+The Build OFED module workflow will check that the filesystem is configured (noexec disabled)
+to allow the DOCA build script to run. The workflow will also install any necessary dependencies
+for the module build.
 
-At the end of the playbook following the kernel module build, the OFED userspace packages
-are downloaded from the upstream repository in order to upload these to Ark.
+The build script will output a ``doca-kernel-repo`` RPM which contains all kernel modules built
+as part of the workflow. When this RPM is installed, the repofile is created pointing to the
+modules in `/usr/share/doca-host-<doca-version>/Modules/<kernel-version>/` on the host.
 
 push-ofed
 ---------
 
-As we're not syncing OFED from any upstream source, and are instead creating our own
-repository of custom packages, we will be required to setup the Pulp distribution/publication
-and upload the content directly to Ark. This playbook uses the Pulp CLI to upload the RPMs
-to Ark.
+As mentioned above, the DOCA repository is synced into the `doca` repository in Ark. This workflow
+will upload the ``doca-kernel-repo`` RPM to a seperate repository named `doca-modules`. The version
+for this repository is set in `pulp-repo-versions.yml` and is disabled for local pulp syncs by
+default.
+
+Install process
+===============
+
+Pre-requisites
+--------------
+
+* Ensure the OFED hosts are upgraded with the latest packages in the point release.
+
+* The bootloader has been configured to use the latest kernel (reset-bls-entries.yml)
+
+* Ensure repositories have been templated by setting:
+
+  .. code-block:: yaml
+
+    dnf_install_doca: true
+
+install-doca
+------------
+
+A playbook is provided to install DOCA on hosts in the `mlnx` group. Ensure this group
+is configured to include the hosts you wish to install DOCA on. To run the install
+playbook:
+
+.. code-block:: console
+
+    kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/install-doca.yml

etc/kayobe/ansible/build-ofed-rocky.yml

@@ -1,5 +1,5 @@
 ---
-- name: Build OFED packages
+- name: Build OFED kernel modules
   become: true
   hosts: ofed-builder
   gather_facts: false
@@ -22,7 +22,6 @@
           - rpm-build
           - automake
           - patch
-          - kernel
           - kernel-devel
           - autoconf
           - pciutils
@@ -37,37 +36,10 @@
           - cmake-filesystem
           - libnl3-devel
           - python3-devel
+          - doca-extra
         state: latest
         update_cache: true
 
-    - name: Add DOCA host repository package
-      ansible.builtin.dnf:
-        name: "https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm"
-        disable_gpg_check: true
-
-    - name: Install DOCA extra packages
-      ansible.builtin.dnf:
-        name: doca-extra
-
-    - name: Create build directory
-      ansible.builtin.file:
-        path: /home/cloud-user/ofed
-        state: directory
-        mode: "0777"
-
-    - name: Set build directory
-      ansible.builtin.replace:
-        path: /opt/mellanox/doca/tools/doca-kernel-support
-        regexp: TMP_DIR=\$1
-        replace: TMP_DIR=/home/cloud-user/ofed
-
     - name: Build OFED kernel modules
-      ansible.builtin.shell:
-        cmd: |
-          /opt/mellanox/doca/tools/doca-kernel-support
-
-    - name: Download OFED userspace packages
-      ansible.builtin.dnf:
-        name: doca-ofed-userspace
-        download_only: true
-        download_dir: /home/cloud-user/ofed
+      ansible.builtin.command:
+        cmd: /opt/mellanox/doca/tools/doca-kernel-support

etc/kayobe/ansible/install-doca.yml

@@ -0,0 +1,28 @@
+---
+- name: Install DOCA
+  become: true
+  hosts: mlnx
+  gather_facts: true
+  tasks:
+    - name: Get running kernel
+      ansible.builtin.command:
+        cmd: "uname -r"
+      register: kernel
+
+    - name: Install kernel repo
+      ansible.builtin.dnf:
+        name: doca-kernel-repo
+        state: latest
+        update_cache: true
+
+    - name: Ensure correct priority for DOCA modules
+      ansible.builtin.lineinfile:
+        line: "priority=-2"
+        insertafter: EOF
+        path: "/etc/yum.repos.d/doca-kernel-{{ kernel.stdout }}.repo"
+
+    - name: Install DOCA OFED
+      ansible.builtin.dnf:
+        name: doca-ofed
+        state: latest
+        update_cache: true

etc/kayobe/ansible/push-ofed.yml

@@ -1,45 +1,71 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
+  vars:
+    venv: "/opt/kayobe/venvs/kayobe"
+    doca_extract_path: "/home/stack/doca"
   tasks:
+    - name: Get OFED module repo variables
+      ansible.builtin.set_fact:
+        doca_modules_repo_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='name') | join('') }}"
+        doca_modules_repo_base_path: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='base_path') | join('') }}"
+        doca_modules_repo_distribution_name: "{{ stackhpc_pulp_rpm_repos | selectattr('name', 'search', 'OFED') | map(attribute='distribution_name') | join('') }}"
+
     - name: Install python dependencies
       ansible.builtin.pip:
         name: pulp-cli
 
-    - name: Create Pulp repository for OFED
+    - name: Ensure Pulp configuration directory exists
+      ansible.builtin.file:
+        path: /home/stack/.config/pulp/
+        state: directory
+        recurse: true
+
+    - name: Setup Pulp credentials
+      ansible.builtin.blockinfile:
+        path: /home/stack/.config/pulp/cli.toml
+        create: true
+        block: |
+          [cli]
+          base_url = '{{ stackhpc_release_pulp_url }}'
+          verify_ssl = true
+          format = "json"
+          username = '{{ stackhpc_release_pulp_username }}'
+          password = '{{ stackhpc_release_pulp_password }}'
+      no_log: true
+
+    - name: Find DOCA kernel repo package
+      ansible.builtin.shell:
+        cmd: 'find /tmp/DOCA.* -name doca-kernel-repo-*'
+      register: doca_kernel_repo
+      changed_when: false
+
+    - name: Create Pulp repository for DOCA kernel modules
       pulp.squeezer.rpm_repository:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
+        name: "{{ doca_modules_repo_name }}"
         state: present
       retries: "{{ pulp_timeout_retries | default(3) }}"
 
-    - name: Lookup Pulp RPMs on builder
-      ansible.builtin.find:
-        paths: /home/cloud-user/ofed
-      register: rpm_dir
-
     - name: Upload OFED RPMs to Pulp
       ansible.builtin.shell:
         cmd: |
-          pulp \
-          --base-url '{{ stackhpc_release_pulp_url }}' \
-          --username '{{ stackhpc_release_pulp_username }}' \
-          --password '{{ stackhpc_release_pulp_password }}' \
+          {{ venv }}/bin/pulp \
           rpm content \
           --type package upload \
-          --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
-          --file {{ item.path }} \
-      with_items: "{{ rpm_dir.files }}"
-      no_log: true
+          --repository '{{ doca_modules_repo_name }}' \
+          --file {{ item }} \
+          --no-publish \
+      with_items: "{{ doca_kernel_repo.stdout_lines }}"
 
     - name: Create Pulp publication for OFED
       pulp.squeezer.rpm_publication:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        repository: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}"
+        repository: "{{ doca_modules_repo_name }}"
         state: present
       register: publication
 
@@ -48,7 +74,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}"
+        name: "{{ doca_modules_repo_distribution_name + ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}"
+        content_guard: development
+        base_path: "{{ doca_modules_repo_base_path + ofed_tag }}"
         state: present