From b564944939cfda2dfb9a30e41847bb54a75c0ae2 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 8 Oct 2024 10:46:35 +0100
Subject: [PATCH 01/28] Update working directory

---
 .github/workflows/package-build-ofed.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 798e0c4bf..352a4f684 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -86,6 +86,7 @@ jobs:
         id: image_tag
         run: |
           echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
+        working-directory: ${{ github.workspace }}/src/kayobe-config
 
       # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
       - name: Output image name

From c317e838b5f12d0e915f3c56e77b11979311c0cd Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 8 Oct 2024 14:19:29 +0100
Subject: [PATCH 02/28] Update bootstrap user

---
 .github/workflows/package-build-ofed.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 352a4f684..1301ee5ed 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -192,7 +192,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker
+          kayobe seed host configure --skip-tags network,docker -e seed_bootstrap_user="cloud-user"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

From 255f226f159795bd011a453240e2f6053e80551e Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 8 Oct 2024 14:31:44 +0100
Subject: [PATCH 03/28] Disable local docker registry

---
 .github/workflows/package-build-ofed.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 1301ee5ed..09d2658f9 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -192,7 +192,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host configure --skip-tags network,docker -e seed_bootstrap_user="cloud-user"
+          kayobe seed host configure --skip-tags network,docker,docker-registry -e seed_bootstrap_user="cloud-user"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

From ebd8420ebe2bf6f7371792409484868d1f2f9920 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 9 Oct 2024 09:03:58 +0100
Subject: [PATCH 04/28] Assume yes for update

---
 .github/workflows/package-build-ofed.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 09d2658f9..1407198aa 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -200,7 +200,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed host command run --become --command "dnf distro-sync --refresh"
+          kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 

From ce0490e56971ffbabdc559b0abf399516a9f7558 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 21 Oct 2024 14:11:30 +0100
Subject: [PATCH 05/28] Fix unknown terminal type

---
 .github/workflows/package-build-ofed.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 1407198aa..3a8b6e465 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -236,6 +236,7 @@ jobs:
           kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+          TERM: linux
 
       - name: Run OFED upload playbook
         run: |

From 744bc13639194d291f3db677958c8639b1b587d6 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 24 Oct 2024 11:16:43 +0100
Subject: [PATCH 06/28] Try include env

---
 .github/workflows/package-build-ofed.yml | 2 +-
 etc/kayobe/ansible/build-ofed-rocky.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 3a8b6e465..94564b717 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -233,7 +233,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml -vvv
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
           TERM: linux
diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index 4c5b74bba..f41cd7439 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -64,7 +64,7 @@
     - name: Build OFED kernel modules
       ansible.builtin.shell:
         cmd: |
-          /opt/mellanox/doca/tools/doca-kernel-support
+          TERM=linux /opt/mellanox/doca/tools/doca-kernel-support
 
     - name: Download OFED userspace packages
       ansible.builtin.dnf:

From 42b6b7314069d2811cf027bad01ce151ab1e40ca Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 13:26:54 +0000
Subject: [PATCH 07/28] Tag in LVM

---
 etc/kayobe/environments/ci-builder/seed.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/environments/ci-builder/seed.yml b/etc/kayobe/environments/ci-builder/seed.yml
index 5054fe36e..8599de627 100644
--- a/etc/kayobe/environments/ci-builder/seed.yml
+++ b/etc/kayobe/environments/ci-builder/seed.yml
@@ -9,5 +9,5 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl
 # Uncomment this if using an LVM-based image.
 # Seed lvm configuration. See inventory/group_vars/seed/lvm.yml
 # for the exact configuration.
-# seed_lvm_groups:
-#   - "{{ stackhpc_lvm_group_rootvg }}"
+seed_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"

From 90e0c14502b63ea66e33138787232c8530ae4920 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 15:12:09 +0000
Subject: [PATCH 08/28] Enable DNF repos

---
 .../ci-builder/inventory/group_vars/seed/stackhpc-repos        | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos

diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos
new file mode 100644
index 000000000..107f75c1c
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos
@@ -0,0 +1,3 @@
+---
+
+stackhpc_repos_enabled: true

From ecd1bcbb4c4c65c4e89fef3b2634308d2f56d616 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 16:21:10 +0000
Subject: [PATCH 09/28] Enable release credentials

---
 etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index e7f755d59..0b3849f46 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -46,6 +46,10 @@ stackhpc_repo_mirror_password: !vault |
   3837336661653962340a316631366463623138623530373133336665376433633437306631383666
   30333461333535363433363336663664316634343432633766346564323833346663
 
+# Use AIO credentials to push OFED RPMs
+stackhpc_release_pulp_username: "{{ stackhpc_repo_mirror_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_repo_mirror_password }}"
+
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
 stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"

From 80e230f2c0f4ed76ebcd8bc4a9a2ccd770e11496 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 17:26:49 +0000
Subject: [PATCH 10/28] Use image repository credentials instead

---
 etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
index 0b3849f46..b60d7a428 100644
--- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
+++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -46,10 +46,6 @@ stackhpc_repo_mirror_password: !vault |
   3837336661653962340a316631366463623138623530373133336665376433633437306631383666
   30333461333535363433363336663664316634343432633766346564323833346663
 
-# Use AIO credentials to push OFED RPMs
-stackhpc_release_pulp_username: "{{ stackhpc_repo_mirror_username }}"
-stackhpc_release_pulp_password: "{{ stackhpc_repo_mirror_password }}"
-
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
 stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
@@ -99,3 +95,7 @@ stackhpc_docker_registry_password: !vault |
 # Username and password of the overcloud host image repository.
 stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}"
+
+# Use image credentials to push OFED RPMs
+stackhpc_release_pulp_username: "{{ stackhpc_image_repository_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_image_repository_password }}"

From 8e223dae90807bd3183882070378686ff4641d17 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 18:32:38 +0000
Subject: [PATCH 11/28] Switch to stack user

---
 etc/kayobe/ansible/build-ofed-rocky.yml | 6 +++---
 etc/kayobe/ansible/push-ofed.yml        | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index f41cd7439..fe6c2f432 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -51,7 +51,7 @@
 
     - name: Create build directory
       ansible.builtin.file:
-        path: /home/cloud-user/ofed
+        path: /home/stack/ofed
         state: directory
         mode: 0777
 
@@ -59,7 +59,7 @@
       ansible.builtin.replace:
         path: /opt/mellanox/doca/tools/doca-kernel-support
         regexp: 'TMP_DIR=\$1'
-        replace: 'TMP_DIR=/home/cloud-user/ofed'
+        replace: 'TMP_DIR=/home/stack/ofed'
 
     - name: Build OFED kernel modules
       ansible.builtin.shell:
@@ -70,4 +70,4 @@
       ansible.builtin.dnf:
         name: doca-ofed-userspace
         download_only: true
-        download_dir: /home/cloud-user/ofed
+        download_dir: /home/stack/ofed
diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index c0214a0b0..cbffd114d 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -17,7 +17,7 @@
 
     - name: Lookup Pulp RPMs on builder
       ansible.builtin.find:
-        paths: "/home/cloud-user/ofed"
+        paths: "/home/stack/ofed"
       register: rpm_dir
 
     - name: Upload OFED RPMs to Pulp

From a736c44b7eeced7aa90efccd6101cf2f722216ab Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 19:06:43 +0000
Subject: [PATCH 12/28] Remove debug options

---
 .github/workflows/package-build-ofed.yml | 2 +-
 etc/kayobe/ansible/build-ofed-rocky.yml  | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 94564b717..3a8b6e465 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -233,7 +233,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml -vvv
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
           TERM: linux
diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index fe6c2f432..08c786f52 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -22,7 +22,6 @@
           - rpm-build
           - automake
           - patch
-          - kernel
           - kernel-devel
           - autoconf
           - pciutils

From e46e47ee88a774315c54b662657120bcae14ba2e Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 29 Oct 2024 14:06:43 +0000
Subject: [PATCH 13/28] Use venv

---
 etc/kayobe/ansible/push-ofed.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index cbffd114d..4feb3049b 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -1,6 +1,8 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
+  vars:
+    venv: "/opt/kayobe/venvs/kayobe"
   tasks:
     - name: Install python dependencies
       ansible.builtin.pip:
@@ -23,7 +25,7 @@
     - name: Upload OFED RPMs to Pulp
       ansible.builtin.shell:
         cmd: |
-          pulp \
+          {{ venv }}/bin/pulp \
           --base-url '{{ stackhpc_release_pulp_url }}' \
           --username '{{ stackhpc_release_pulp_username }}' \
           --password '{{ stackhpc_release_pulp_password }}' \

From fdab85cc263c5520ef60e4672e23d580a59aa2dc Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 30 Oct 2024 14:22:07 +0000
Subject: [PATCH 14/28] Stop publishing on content upload

---
 etc/kayobe/ansible/push-ofed.yml | 10 ++++++++--
 etc/kayobe/ofed.yml              |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index 4feb3049b..db4a83a0d 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -8,6 +8,10 @@
       ansible.builtin.pip:
         name: pulp-cli
 
+    - name: Generate timestamp for distribution
+      ansible.builtin.set_fact:
+        stackhpc_doca_ofed_repo_version: "{{ ansible_date_time.iso8601_basic_short }}"
+
     - name: Create Pulp repository for OFED
       pulp.squeezer.rpm_repository:
         pulp_url: "{{ stackhpc_release_pulp_url }}"
@@ -33,6 +37,7 @@
           --type package upload \
           --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
           --file {{ item.path }} \
+          --no-publish \
       with_items: "{{ rpm_dir.files }}"
       no_log: true
 
@@ -50,7 +55,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }} ~ {{ stackhpc_doca_ofed_repo_version }}"
         publication: "{{ publication.publication.pulp_href }}"
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}"
+        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ stackhpc_doca_ofed_repo_version }}"
+        content_guard: "development"
         state: present
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 696e3c93b..e78d512bd 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -8,5 +8,5 @@ stackhpc_pulp_doca_ofed_version: 24.07
 stackhpc_pulp_repo_doca_ofed_rhel9:
   name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
   url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
+  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
   base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"

From 9c61a4a3002c49efd05a051b67d7e1252a46c89a Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 30 Oct 2024 14:37:14 +0000
Subject: [PATCH 15/28] Move OFED config to group_vars

---
 .../ci-builder/inventory/group_vars/ofed-builder/lvm          | 4 ++++
 .../group_vars/{seed => ofed-builder}/stackhpc-repos          | 0
 etc/kayobe/environments/ci-builder/seed.yml                   | 4 ++--
 3 files changed, 6 insertions(+), 2 deletions(-)
 create mode 100644 etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm
 rename etc/kayobe/environments/ci-builder/inventory/group_vars/{seed => ofed-builder}/stackhpc-repos (100%)

diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm
new file mode 100644
index 000000000..6d3e612f3
--- /dev/null
+++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm
@@ -0,0 +1,4 @@
+---
+
+seed_lvm_groups:
+  - "{{ stackhpc_lvm_group_rootvg }}"
diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos
similarity index 100%
rename from etc/kayobe/environments/ci-builder/inventory/group_vars/seed/stackhpc-repos
rename to etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos
diff --git a/etc/kayobe/environments/ci-builder/seed.yml b/etc/kayobe/environments/ci-builder/seed.yml
index 8599de627..5054fe36e 100644
--- a/etc/kayobe/environments/ci-builder/seed.yml
+++ b/etc/kayobe/environments/ci-builder/seed.yml
@@ -9,5 +9,5 @@ seed_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' else 'cl
 # Uncomment this if using an LVM-based image.
 # Seed lvm configuration. See inventory/group_vars/seed/lvm.yml
 # for the exact configuration.
-seed_lvm_groups:
-  - "{{ stackhpc_lvm_group_rootvg }}"
+# seed_lvm_groups:
+#   - "{{ stackhpc_lvm_group_rootvg }}"

From f04c3852df0e9030eab34fd17de8470d20a53c65 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 30 Oct 2024 15:27:15 +0000
Subject: [PATCH 16/28] try gather facts

---
 etc/kayobe/ansible/build-ofed-rocky.yml | 5 -----
 etc/kayobe/ansible/push-ofed.yml        | 1 +
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index 08c786f52..05faa828e 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -60,11 +60,6 @@
         regexp: 'TMP_DIR=\$1'
         replace: 'TMP_DIR=/home/stack/ofed'
 
-    - name: Build OFED kernel modules
-      ansible.builtin.shell:
-        cmd: |
-          TERM=linux /opt/mellanox/doca/tools/doca-kernel-support
-
     - name: Download OFED userspace packages
       ansible.builtin.dnf:
         name: doca-ofed-userspace
diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index db4a83a0d..f75b3eb55 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -1,6 +1,7 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
+  gather_facts: true
   vars:
     venv: "/opt/kayobe/venvs/kayobe"
   tasks:

From d49212c708e8ca25e7adb85d50934f5371ff9124 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Wed, 30 Oct 2024 15:49:39 +0000
Subject: [PATCH 17/28] Use strftime instead

---
 etc/kayobe/ansible/build-ofed-rocky.yml | 5 +++++
 etc/kayobe/ansible/push-ofed.yml        | 5 ++---
 etc/kayobe/ofed.yml                     | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index 05faa828e..08c786f52 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -60,6 +60,11 @@
         regexp: 'TMP_DIR=\$1'
         replace: 'TMP_DIR=/home/stack/ofed'
 
+    - name: Build OFED kernel modules
+      ansible.builtin.shell:
+        cmd: |
+          TERM=linux /opt/mellanox/doca/tools/doca-kernel-support
+
     - name: Download OFED userspace packages
       ansible.builtin.dnf:
         name: doca-ofed-userspace
diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index f75b3eb55..afc3b032a 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -1,7 +1,6 @@
 ---
 - name: Push OFED packages
   hosts: ofed-builder
-  gather_facts: true
   vars:
     venv: "/opt/kayobe/venvs/kayobe"
   tasks:
@@ -11,7 +10,7 @@
 
     - name: Generate timestamp for distribution
       ansible.builtin.set_fact:
-        stackhpc_doca_ofed_repo_version: "{{ ansible_date_time.iso8601_basic_short }}"
+        stackhpc_doca_ofed_repo_version: "{{ '%Y%m%dT%H%M%S' | strftime }}"
 
     - name: Create Pulp repository for OFED
       pulp.squeezer.rpm_repository:
@@ -56,7 +55,7 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }} ~ {{ stackhpc_doca_ofed_repo_version }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}-{{ stackhpc_doca_ofed_repo_version }}"
         publication: "{{ publication.publication.pulp_href }}"
         base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ stackhpc_doca_ofed_repo_version }}"
         content_guard: "development"
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index e78d512bd..696e3c93b 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -8,5 +8,5 @@ stackhpc_pulp_doca_ofed_version: 24.07
 stackhpc_pulp_repo_doca_ofed_rhel9:
   name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
   url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
+  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
   base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"

From 3abbe4076585ab6144acab86bb615670f033073b Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 18:06:07 +0000
Subject: [PATCH 18/28] Add Pulp workflow support

---
 .github/workflows/package-build-ofed.yml |  8 +++++++-
 etc/kayobe/ansible/push-ofed.yml         |  6 +++---
 etc/kayobe/dnf.yml                       | 13 +++++++++++++
 etc/kayobe/stackhpc.yml                  |  4 ++++
 4 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 3a8b6e465..7c32fb104 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -48,6 +48,11 @@ jobs:
           BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
           echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
 
+      - name: Generate OFED tag
+        id: ofed_tag
+        run: |
+          echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
+
       - name: Clone StackHPC Kayobe repository
         uses: actions/checkout@v4
         with:
@@ -242,7 +247,8 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml
+          kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \
+          -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}"
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index afc3b032a..507374c5e 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -55,8 +55,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}-{{ stackhpc_doca_ofed_repo_version }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}-{{ ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ stackhpc_doca_ofed_repo_version }}"
-        content_guard: "development"
+        content_guard: development
+        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }/{{ ofed_tag }}"
         state: present
diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index 916ffe5a1..41b12ba31 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -49,6 +49,17 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # file.
 stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"
 
+# OFED repository for Rocky 9
+dnf_custom_repos_ofed:
+  ofed:
+    baseurl: "{{ stackhpc_repo_rocky_ofed_url }}"
+    description: "Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
+    enabled: "{{ dnf_enable_rocky_ofed | bool }}"
+    file: ofed
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+
 # Custom repositories shared between all RHEL 9 derivatives.
 dnf_custom_repos_el9:
   epel:
@@ -152,6 +163,8 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
+# Whether to enable OFED repositories
+dnf_enable_rocky_ofed: false
 
 ###############################################################################
 # DNF Automatic configuration.
diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml
index 8526eb7cd..504fff3e2 100644
--- a/etc/kayobe/stackhpc.yml
+++ b/etc/kayobe/stackhpc.yml
@@ -52,6 +52,10 @@ stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}"
 ###############################################################################
 # RPMs
 
+# OFED
+stackhpc_repo_ofed_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca_ofed/{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rocky_9_doca_ofed_version }}"
+stackhpc_repo_rocky_9_doca_ofed_version: "{{ stackhpc_repo_distribution }}"
+
 # Grafana
 stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/oss/rpm/{{ stackhpc_repo_grafana_version }}"
 stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}"

From f5651dd22857ea616ad1b887c8aa3b741bbf9f4e Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Mon, 28 Oct 2024 18:24:08 +0000
Subject: [PATCH 19/28] Use defined versions for OFED

---
 etc/kayobe/ansible/build-ofed-rocky.yml | 2 +-
 etc/kayobe/ofed.yml                     | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index 08c786f52..574a46739 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -41,7 +41,7 @@
 
     - name: Add DOCA host repository package
       ansible.builtin.dnf:
-        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
+        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_{{ stackhpc_pulp_doca_version }}/doca-host-{{ stackhpc_pulp_doca_host_version }}-{{ stackhpc_pulp_doca_host_minor_version }}_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
         disable_gpg_check: true
 
     - name: Install DOCA extra packages
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 696e3c93b..4b9526f5b 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -1,6 +1,13 @@
 ---
 # DOCA OFED configuration
 
+# DOCA version
+stackhpc_pulp_doca_version: 2.8
+
+# DOCA host version
+stackhpc_pulp_doca_host_version: 2.8.0
+stackhpc_pulp_doca_host_minor_version: 204000
+
 # DOCA OFED version
 stackhpc_pulp_doca_ofed_version: 24.07
 

From 16ba9e23ef635f944bf215d2ed25c774fd87d0b1 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 1 Nov 2024 09:39:38 +0000
Subject: [PATCH 20/28] Use dash instead

---
 etc/kayobe/ansible/push-ofed.yml | 2 +-
 etc/kayobe/ofed.yml              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index 507374c5e..e868d78b6 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -55,7 +55,7 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}-{{ ofed_tag }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}{{ ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
         content_guard: development
         base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }/{{ ofed_tag }}"
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 4b9526f5b..25412fd9c 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -15,5 +15,5 @@ stackhpc_pulp_doca_ofed_version: 24.07
 stackhpc_pulp_repo_doca_ofed_rhel9:
   name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
   url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
+  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
   base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"

From d7c14f177c4f83a1afbe590866584490cb0e7db8 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 1 Nov 2024 10:09:03 +0000
Subject: [PATCH 21/28] Allow local pulp sync for OFED

---
 etc/kayobe/ofed.yml | 18 +++++++++++++++---
 etc/kayobe/pulp.yml |  1 +
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index 25412fd9c..c418ed434 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -1,7 +1,8 @@
 ---
-# DOCA OFED configuration
+# DOCA OFED configuration.
 
-# DOCA version
+###############################################################################
+# DOCA versions
 stackhpc_pulp_doca_version: 2.8
 
 # DOCA host version
@@ -11,9 +12,20 @@ stackhpc_pulp_doca_host_minor_version: 204000
 # DOCA OFED version
 stackhpc_pulp_doca_ofed_version: 24.07
 
-# DOCA OFED repositories
+###############################################################################
+# Pulp configuration for DOCA OFED
+
+# Whether to sync OFED packages into the local Pulp service
+stackhpc_pulp_sync_ofed: false
+
+# OFED repository for Rocky 9
 stackhpc_pulp_repo_doca_ofed_rhel9:
   name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
   url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
   distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
   base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+  required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes
diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index f05a36a8f..81edf1ee5 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -380,6 +380,7 @@ stackhpc_pulp_repository_rpm_repos: >-
   {%- endif -%}
   {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(rpm_repo)) -%}
   {%- endfor -%}
+  {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(stackhpc_pulp_repo_doca_ofed_rhel9)) -%}
   {{ rpm_repos }}
 
 # Publication format is a subset of distribution.

From 6ef98b27addbbe96e60ffd79b62e53f16821277c Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 1 Nov 2024 12:39:02 +0000
Subject: [PATCH 22/28] Allow logs for RPM upload

---
 etc/kayobe/ansible/push-ofed.yml | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml
index e868d78b6..8937195a7 100644
--- a/etc/kayobe/ansible/push-ofed.yml
+++ b/etc/kayobe/ansible/push-ofed.yml
@@ -8,9 +8,24 @@
       ansible.builtin.pip:
         name: pulp-cli
 
-    - name: Generate timestamp for distribution
-      ansible.builtin.set_fact:
-        stackhpc_doca_ofed_repo_version: "{{ '%Y%m%dT%H%M%S' | strftime }}"
+    - name: Ensure Pulp configuration directory exists
+      ansible.builtin.file:
+        path: /home/stack/.config/pulp/
+        state: directory
+        recurse: true
+
+    - name: Setup Pulp credentials
+      ansible.builtin.blockinfile:
+        path: /home/stack/.config/pulp/cli.toml
+        create: true
+        block: |
+          [cli]
+          base_url = '{{ stackhpc_release_pulp_url }}'
+          verify_ssl = true
+          format = "json"
+          username = '{{ stackhpc_release_pulp_username }}'
+          password = '{{ stackhpc_release_pulp_password }}'
+      no_log: true
 
     - name: Create Pulp repository for OFED
       pulp.squeezer.rpm_repository:
@@ -30,16 +45,12 @@
       ansible.builtin.shell:
         cmd: |
           {{ venv }}/bin/pulp \
-          --base-url '{{ stackhpc_release_pulp_url }}' \
-          --username '{{ stackhpc_release_pulp_username }}' \
-          --password '{{ stackhpc_release_pulp_password }}' \
           rpm content \
           --type package upload \
           --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \
           --file {{ item.path }} \
           --no-publish \
       with_items: "{{ rpm_dir.files }}"
-      no_log: true
 
     - name: Create Pulp publication for OFED
       pulp.squeezer.rpm_publication:
@@ -55,8 +66,8 @@
         pulp_url: "{{ stackhpc_release_pulp_url }}"
         username: "{{ stackhpc_release_pulp_username }}"
         password: "{{ stackhpc_release_pulp_password }}"
-        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}{{ ofed_tag }}"
+        name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name + ofed_tag }}"
         publication: "{{ publication.publication.pulp_href }}"
         content_guard: development
-        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }/{{ ofed_tag }}"
+        base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ ofed_tag }}"
         state: present

From 2e1d7c90c6cd0d23e95bb6200cb7101bed9a07a8 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 1 Nov 2024 12:42:12 +0000
Subject: [PATCH 23/28] Fix DNF config

---
 etc/kayobe/dnf.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index 41b12ba31..47cdd2845 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -47,7 +47,7 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
-stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}"
+stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool | combine(dnf_custom_repos_ofed if dnf_install_rocky_ofed) | bool else {}) }}"
 
 # OFED repository for Rocky 9
 dnf_custom_repos_ofed:
@@ -163,9 +163,13 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
-# Whether to enable OFED repositories
+# Whether to enable OFED repositories. This affects RedHat-based systems only.
 dnf_enable_rocky_ofed: false
 
+# Whether to create a repo file for OFED. This affects RedHat-based
+# systems only.
+dnf_install_rocky_ofed: false
+
 ###############################################################################
 # DNF Automatic configuration.
 

From 75cde557add010e2e8ed1c680574f62e3325b78c Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Fri, 1 Nov 2024 12:53:34 +0000
Subject: [PATCH 24/28] Fix DNF again

---
 etc/kayobe/dnf.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index 47cdd2845..76f551cce 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -47,7 +47,7 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
-stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool | combine(dnf_custom_repos_ofed if dnf_install_rocky_ofed) | bool else {}) }}"
+stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_ofed if dnf_install_rocky_ofed | bool else {}) }}"
 
 # OFED repository for Rocky 9
 dnf_custom_repos_ofed:
@@ -143,6 +143,9 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}"
 # Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only.
 dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}"
 
+# Whether to enable OFED repositories. This affects RedHat-based systems only.
+dnf_enable_rocky_ofed: "{{ dnf_install_rocky_ofed | bool }}"
+
 # URL of EPEL GPG keys.
 dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
 
@@ -163,9 +166,6 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
-# Whether to enable OFED repositories. This affects RedHat-based systems only.
-dnf_enable_rocky_ofed: false
-
 # Whether to create a repo file for OFED. This affects RedHat-based
 # systems only.
 dnf_install_rocky_ofed: false

From 788731ce502e39355a9582176732e2cb43b35a32 Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 7 Nov 2024 17:00:02 +0000
Subject: [PATCH 25/28] Add DOCA Repo to local pulp sync

---
 etc/kayobe/pulp.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 81edf1ee5..887bbd8fe 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -365,6 +365,12 @@ stackhpc_pulp_rpm_repos:
     base_path: "opensearch-dashboards/2.x/yum/"
     required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
+  - name: DOCA Online Repo 2.8.0
+    url: "{{ stackhpc_release_pulp_content_url }}/doca/2.8.0/rhel9.4/x86_64/{{ stackhpc_pulp_repo_doca_version }}"
+    distribution_name: "doca-"
+    base_path: "doca/2.8.0/rhel9.4/x86_64/"
+    required: "{{ stackhpc_pulp_sync_el_9 }}"
+
 # RPM repositories
 stackhpc_pulp_repository_rpm_repos: >-
   {%- set rpm_repos = [] -%}

From 594b8988fe0625bad1842eba3118408e648b31fd Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Thu, 7 Nov 2024 17:02:41 +0000
Subject: [PATCH 26/28] Use host version variable

---
 etc/kayobe/pulp.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml
index 887bbd8fe..fd7f50e2e 100644
--- a/etc/kayobe/pulp.yml
+++ b/etc/kayobe/pulp.yml
@@ -365,10 +365,10 @@ stackhpc_pulp_rpm_repos:
     base_path: "opensearch-dashboards/2.x/yum/"
     required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
-  - name: DOCA Online Repo 2.8.0
-    url: "{{ stackhpc_release_pulp_content_url }}/doca/2.8.0/rhel9.4/x86_64/{{ stackhpc_pulp_repo_doca_version }}"
+  - name: DOCA Online Repo {{ stackhpc_pulp_doca_host_version }}
+    url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/{{ stackhpc_pulp_repo_doca_version }}"
     distribution_name: "doca-"
-    base_path: "doca/2.8.0/rhel9.4/x86_64/"
+    base_path: "doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/"
     required: "{{ stackhpc_pulp_sync_el_9 }}"
 
 # RPM repositories

From 1919f22c4d08fc50913d40b18487a3b282cfabeb Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 12 Nov 2024 16:41:06 +0000
Subject: [PATCH 27/28] Only build kernel modules

---
 .github/workflows/package-build-ofed.yml |  4 ++--
 etc/kayobe/ansible/build-ofed-rocky.yml  | 10 ++--------
 etc/kayobe/ofed.yml                      | 23 ++++++++++-------------
 3 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml
index 7c32fb104..2e15d8bba 100644
--- a/.github/workflows/package-build-ofed.yml
+++ b/.github/workflows/package-build-ofed.yml
@@ -1,5 +1,5 @@
 ---
-name: Build OFED packages
+name: Build OFED kernel modules
 on:
   workflow_dispatch:
     inputs:
@@ -23,7 +23,7 @@ env:
   KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 jobs:
   overcloud-ofed-packages:
-    name: Build OFED packages
+    name: Build OFED kernel modules
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     runs-on: arc-skc-host-image-builder-runner
     permissions: {}
diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml
index 574a46739..160564e67 100644
--- a/etc/kayobe/ansible/build-ofed-rocky.yml
+++ b/etc/kayobe/ansible/build-ofed-rocky.yml
@@ -1,5 +1,5 @@
 ---
-- name: Build OFED packages
+- name: Build OFED kernel modules
   become: true
   hosts: ofed-builder
   gather_facts: false
@@ -41,7 +41,7 @@
 
     - name: Add DOCA host repository package
       ansible.builtin.dnf:
-        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_{{ stackhpc_pulp_doca_version }}/doca-host-{{ stackhpc_pulp_doca_host_version }}-{{ stackhpc_pulp_doca_host_minor_version }}_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
+        name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_{{ stackhpc_pulp_doca_version[:2] }}/doca-host-{{ stackhpc_pulp_doca_version }}-{{ stackhpc_pulp_doca_host_minor_version }}_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm
         disable_gpg_check: true
 
     - name: Install DOCA extra packages
@@ -64,9 +64,3 @@
       ansible.builtin.shell:
         cmd: |
           TERM=linux /opt/mellanox/doca/tools/doca-kernel-support
-
-    - name: Download OFED userspace packages
-      ansible.builtin.dnf:
-        name: doca-ofed-userspace
-        download_only: true
-        download_dir: /home/stack/ofed
diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml
index c418ed434..c5e6a3b4b 100644
--- a/etc/kayobe/ofed.yml
+++ b/etc/kayobe/ofed.yml
@@ -2,12 +2,9 @@
 # DOCA OFED configuration.
 
 ###############################################################################
-# DOCA versions
-stackhpc_pulp_doca_version: 2.8
-
 # DOCA host version
-stackhpc_pulp_doca_host_version: 2.8.0
-stackhpc_pulp_doca_host_minor_version: 204000
+stackhpc_pulp_doca_version: 2.8.0
+stackhpc_pulp_doca_minor_version: 204000
 
 # DOCA OFED version
 stackhpc_pulp_doca_ofed_version: 24.07
@@ -15,16 +12,16 @@ stackhpc_pulp_doca_ofed_version: 24.07
 ###############################################################################
 # Pulp configuration for DOCA OFED
 
-# Whether to sync OFED packages into the local Pulp service
-stackhpc_pulp_sync_ofed: false
+# Whether to sync OFED kernel module packages into the local Pulp service
+stackhpc_pulp_sync_ofed_modules: false
 
-# OFED repository for Rocky 9
+# OFED kernel module repository for Rocky 9
 stackhpc_pulp_repo_doca_ofed_rhel9:
-  name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
-  url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}-"
-  base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
-  required: "{{ stackhpc_pulp_sync_ofed | bool and stackhpc_pulp_sync_el_9 | bool }}"
+  name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}
+  url: "{{ stackhpc_release_pulp_content_url }}/doca_modules/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64"
+  distribution_name: "doca-modules-"
+  base_path: "doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/"
+  required: "{{ stackhpc_pulp_sync_ofed_modules | bool and stackhpc_pulp_sync_el_9 | bool }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

From 37956e7ceed36d2a2cb811935637499bb73da50c Mon Sep 17 00:00:00 2001
From: Jake Hutchinson <jake@stackhpc.com>
Date: Tue, 12 Nov 2024 16:53:38 +0000
Subject: [PATCH 28/28] Setup DNF config for DOCA

---
 etc/kayobe/dnf.yml | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml
index 76f551cce..eaeb1d8e4 100644
--- a/etc/kayobe/dnf.yml
+++ b/etc/kayobe/dnf.yml
@@ -47,15 +47,23 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [
 # To use these repos, set stackhpc_repos_enabled to true.
 # This is done by default for hosts in the overcloud group via a group_vars
 # file.
-stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_ofed if dnf_install_rocky_ofed | bool else {}) }}"
-
-# OFED repository for Rocky 9
-dnf_custom_repos_ofed:
-  ofed:
-    baseurl: "{{ stackhpc_repo_rocky_ofed_url }}"
-    description: "Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}"
-    enabled: "{{ dnf_enable_rocky_ofed | bool }}"
-    file: ofed
+stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_ofed if dnf_install_doca | bool else {}) }}"
+
+# DOCA repositories
+dnf_custom_repos_doca:
+  doca:
+    baseurl: "{{ stackhpc_repo_rocky_doca_url }}"
+    description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }}"
+    enabled: "{{ dnf_enable_doca | bool }}"
+    file: doca
+    gpgcheck: no
+    username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
+    password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
+  doca-modules:
+    baseurl: "{{ stackhpc_repo_rocky_doca_modules_url }}"
+    description: "DOCA Modules Repo {{ stackhpc_pulp_doca_version }}"
+    enabled: "{{ dnf_enable_doca | bool and dnf_enable_doca_modules | bool }}"
+    file: doca-modules
     gpgcheck: no
     username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}"
     password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}"
@@ -143,8 +151,11 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}"
 # Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only.
 dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}"
 
-# Whether to enable OFED repositories. This affects RedHat-based systems only.
-dnf_enable_rocky_ofed: "{{ dnf_install_rocky_ofed | bool }}"
+# Whether to enable DOCA repositories. This affects RedHat-based systems only.
+dnf_enable_doca: "{{ dnf_install_doca | bool }}"
+
+# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only.
+dnf_enable_doca_modules: "{{ dnf_install_doca_modules | bool }}"
 
 # URL of EPEL GPG keys.
 dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9"
@@ -166,9 +177,9 @@ dnf_enable_docker: true
 #URL of docker repo GPG key
 dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg"
 
-# Whether to create a repo file for OFED. This affects RedHat-based
+# Whether to create a repo file for DOCA. This affects RedHat-based
 # systems only.
-dnf_install_rocky_ofed: false
+dnf_install_doca: false
 
 ###############################################################################
 # DNF Automatic configuration.