diff --git a/.github/workflows/package-build-ofed.yml b/.github/workflows/package-build-ofed.yml index 798e0c4bf..2e15d8bba 100644 --- a/.github/workflows/package-build-ofed.yml +++ b/.github/workflows/package-build-ofed.yml @@ -1,5 +1,5 @@ --- -name: Build OFED packages +name: Build OFED kernel modules on: workflow_dispatch: inputs: @@ -23,7 +23,7 @@ env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} jobs: overcloud-ofed-packages: - name: Build OFED packages + name: Build OFED kernel modules if: github.repository == 'stackhpc/stackhpc-kayobe-config' runs-on: arc-skc-host-image-builder-runner permissions: {} @@ -48,6 +48,11 @@ jobs: BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT + - name: Generate OFED tag + id: ofed_tag + run: | + echo "ofed_tag=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT + - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 with: @@ -86,6 +91,7 @@ jobs: id: image_tag run: | echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/src/kayobe-config # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag - name: Output image name @@ -191,7 +197,7 @@ jobs: run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure --skip-tags network,docker + kayobe seed host configure --skip-tags network,docker,docker-registry -e seed_bootstrap_user="cloud-user" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} @@ -199,7 +205,7 @@ jobs: run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host command run --become --command "dnf distro-sync --refresh" + kayobe seed host command run --become --command "dnf distro-sync --refresh --assumeyes" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} @@ -235,12 +241,14 @@ jobs: kayobe playbook run src/kayobe-config/etc/kayobe/ansible/build-ofed-rocky.yml env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + TERM: linux - name: Run OFED upload playbook run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml + kayobe playbook run src/kayobe-config/etc/kayobe/ansible/push-ofed.yml \ + -e "ofed_tag=${{ steps.ofed_tag.outputs.ofed_tag }}" env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} diff --git a/etc/kayobe/ansible/build-ofed-rocky.yml b/etc/kayobe/ansible/build-ofed-rocky.yml index 4c5b74bba..160564e67 100644 --- a/etc/kayobe/ansible/build-ofed-rocky.yml +++ b/etc/kayobe/ansible/build-ofed-rocky.yml @@ -1,5 +1,5 @@ --- -- name: Build OFED packages +- name: Build OFED kernel modules become: true hosts: ofed-builder gather_facts: false @@ -22,7 +22,6 @@ - rpm-build - automake - patch - - kernel - kernel-devel - autoconf - pciutils @@ -42,7 +41,7 @@ - name: Add DOCA host repository package ansible.builtin.dnf: - name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_2.8/doca-host-2.8.0-204000_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm + name: https://developer.nvidia.com/downloads/networking/secure/doca-sdk/DOCA_{{ stackhpc_pulp_doca_version[:2] }}/doca-host-{{ stackhpc_pulp_doca_version }}-{{ stackhpc_pulp_doca_host_minor_version }}_{{ stackhpc_pulp_doca_ofed_version }}_rhel9{{ stackhpc_pulp_repo_rocky_9_minor_version }}.x86_64.rpm disable_gpg_check: true - name: Install DOCA extra packages @@ -51,7 +50,7 @@ - name: Create build directory ansible.builtin.file: - path: /home/cloud-user/ofed + path: /home/stack/ofed state: directory mode: 0777 @@ -59,15 +58,9 @@ ansible.builtin.replace: path: /opt/mellanox/doca/tools/doca-kernel-support regexp: 'TMP_DIR=\$1' - replace: 'TMP_DIR=/home/cloud-user/ofed' + replace: 'TMP_DIR=/home/stack/ofed' - name: Build OFED kernel modules ansible.builtin.shell: cmd: | - /opt/mellanox/doca/tools/doca-kernel-support - - - name: Download OFED userspace packages - ansible.builtin.dnf: - name: doca-ofed-userspace - download_only: true - download_dir: /home/cloud-user/ofed + TERM=linux /opt/mellanox/doca/tools/doca-kernel-support diff --git a/etc/kayobe/ansible/push-ofed.yml b/etc/kayobe/ansible/push-ofed.yml index c0214a0b0..8937195a7 100644 --- a/etc/kayobe/ansible/push-ofed.yml +++ b/etc/kayobe/ansible/push-ofed.yml @@ -1,11 +1,32 @@ --- - name: Push OFED packages hosts: ofed-builder + vars: + venv: "/opt/kayobe/venvs/kayobe" tasks: - name: Install python dependencies ansible.builtin.pip: name: pulp-cli + - name: Ensure Pulp configuration directory exists + ansible.builtin.file: + path: /home/stack/.config/pulp/ + state: directory + recurse: true + + - name: Setup Pulp credentials + ansible.builtin.blockinfile: + path: /home/stack/.config/pulp/cli.toml + create: true + block: | + [cli] + base_url = '{{ stackhpc_release_pulp_url }}' + verify_ssl = true + format = "json" + username = '{{ stackhpc_release_pulp_username }}' + password = '{{ stackhpc_release_pulp_password }}' + no_log: true + - name: Create Pulp repository for OFED pulp.squeezer.rpm_repository: pulp_url: "{{ stackhpc_release_pulp_url }}" @@ -17,22 +38,19 @@ - name: Lookup Pulp RPMs on builder ansible.builtin.find: - paths: "/home/cloud-user/ofed" + paths: "/home/stack/ofed" register: rpm_dir - name: Upload OFED RPMs to Pulp ansible.builtin.shell: cmd: | - pulp \ - --base-url '{{ stackhpc_release_pulp_url }}' \ - --username '{{ stackhpc_release_pulp_username }}' \ - --password '{{ stackhpc_release_pulp_password }}' \ + {{ venv }}/bin/pulp \ rpm content \ --type package upload \ --repository '{{ stackhpc_pulp_repo_doca_ofed_rhel9.name }}' \ --file {{ item.path }} \ + --no-publish \ with_items: "{{ rpm_dir.files }}" - no_log: true - name: Create Pulp publication for OFED pulp.squeezer.rpm_publication: @@ -48,7 +66,8 @@ pulp_url: "{{ stackhpc_release_pulp_url }}" username: "{{ stackhpc_release_pulp_username }}" password: "{{ stackhpc_release_pulp_password }}" - name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name }}" + name: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.distribution_name + ofed_tag }}" publication: "{{ publication.publication.pulp_href }}" - base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}" + content_guard: development + base_path: "{{ stackhpc_pulp_repo_doca_ofed_rhel9.base_path }}/{{ ofed_tag }}" state: present diff --git a/etc/kayobe/dnf.yml b/etc/kayobe/dnf.yml index 916ffe5a1..eaeb1d8e4 100644 --- a/etc/kayobe/dnf.yml +++ b/etc/kayobe/dnf.yml @@ -47,7 +47,26 @@ dnf_custom_repos: "{{ stackhpc_dnf_repos if stackhpc_repos_enabled | bool else [ # To use these repos, set stackhpc_repos_enabled to true. # This is done by default for hosts in the overcloud group via a group_vars # file. -stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) }}" +stackhpc_dnf_repos: "{{ dnf_custom_repos_el9 | combine(dnf_custom_repos_rocky_9) | combine(dnf_custom_repos_elrepo_9 if dnf_install_elrepo_9 | bool else {}) | combine(dnf_custom_repos_ofed if dnf_install_doca | bool else {}) }}" + +# DOCA repositories +dnf_custom_repos_doca: + doca: + baseurl: "{{ stackhpc_repo_rocky_doca_url }}" + description: "DOCA Online Repo {{ stackhpc_pulp_doca_version }}" + enabled: "{{ dnf_enable_doca | bool }}" + file: doca + gpgcheck: no + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" + doca-modules: + baseurl: "{{ stackhpc_repo_rocky_doca_modules_url }}" + description: "DOCA Modules Repo {{ stackhpc_pulp_doca_version }}" + enabled: "{{ dnf_enable_doca | bool and dnf_enable_doca_modules | bool }}" + file: doca-modules + gpgcheck: no + username: "{{ stackhpc_repo_mirror_username | default(omit, true) }}" + password: "{{ stackhpc_repo_mirror_password | default(omit, true) }}" # Custom repositories shared between all RHEL 9 derivatives. dnf_custom_repos_el9: @@ -132,6 +151,12 @@ dnf_enable_epel: "{{ dnf_install_epel | bool }}" # Whether to enable the ELRepo repository. This affects RedHat-based, 9.x release systems only. dnf_enable_elrepo_9: "{{ dnf_install_elrepo_9 | bool }}" +# Whether to enable DOCA repositories. This affects RedHat-based systems only. +dnf_enable_doca: "{{ dnf_install_doca | bool }}" + +# Whether to enable the DOCA kernel module repository. This affects RedHat-based systems only. +dnf_enable_doca_modules: "{{ dnf_install_doca_modules | bool }}" + # URL of EPEL GPG keys. dnf_epel_9_gpg_key_url: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-9" @@ -152,6 +177,9 @@ dnf_enable_docker: true #URL of docker repo GPG key dnf_docker_gpg_key_url: "https://download.docker.com/linux/centos/gpg" +# Whether to create a repo file for DOCA. This affects RedHat-based +# systems only. +dnf_install_doca: false ############################################################################### # DNF Automatic configuration. diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm new file mode 100644 index 000000000..6d3e612f3 --- /dev/null +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/lvm @@ -0,0 +1,4 @@ +--- + +seed_lvm_groups: + - "{{ stackhpc_lvm_group_rootvg }}" diff --git a/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos new file mode 100644 index 000000000..107f75c1c --- /dev/null +++ b/etc/kayobe/environments/ci-builder/inventory/group_vars/ofed-builder/stackhpc-repos @@ -0,0 +1,3 @@ +--- + +stackhpc_repos_enabled: true diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index e7f755d59..b60d7a428 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -95,3 +95,7 @@ stackhpc_docker_registry_password: !vault | # Username and password of the overcloud host image repository. stackhpc_image_repository_username: "{{ stackhpc_docker_registry_username }}" stackhpc_image_repository_password: "{{ stackhpc_docker_registry_password }}" + +# Use image credentials to push OFED RPMs +stackhpc_release_pulp_username: "{{ stackhpc_image_repository_username }}" +stackhpc_release_pulp_password: "{{ stackhpc_image_repository_password }}" diff --git a/etc/kayobe/ofed.yml b/etc/kayobe/ofed.yml index 696e3c93b..c5e6a3b4b 100644 --- a/etc/kayobe/ofed.yml +++ b/etc/kayobe/ofed.yml @@ -1,12 +1,28 @@ --- -# DOCA OFED configuration +# DOCA OFED configuration. + +############################################################################### +# DOCA host version +stackhpc_pulp_doca_version: 2.8.0 +stackhpc_pulp_doca_minor_version: 204000 # DOCA OFED version stackhpc_pulp_doca_ofed_version: 24.07 -# DOCA OFED repositories +############################################################################### +# Pulp configuration for DOCA OFED + +# Whether to sync OFED kernel module packages into the local Pulp service +stackhpc_pulp_sync_ofed_modules: false + +# OFED kernel module repository for Rocky 9 stackhpc_pulp_repo_doca_ofed_rhel9: - name: Mellanox Technologies doca_ofed {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} - url: "{{ stackhpc_release_pulp_content_url }}/doca_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64" - distribution_name: "mlnx-ofed-{{ stackhpc_pulp_doca_ofed_version }}-rocky-9-{{ stackhpc_pulp_repo_rocky_9_minor_version }}" - base_path: "mlnx_ofed/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64" + name: OFED Kernel modules for DOCA {{ stackhpc_pulp_doca_ofed_version }} Rocky 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} + url: "{{ stackhpc_release_pulp_content_url }}/doca_modules/{{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64" + distribution_name: "doca-modules-" + base_path: "doca-modules/{{ stackhpc_pulp_doca_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/" + required: "{{ stackhpc_pulp_sync_ofed_modules | bool and stackhpc_pulp_sync_el_9 | bool }}" + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index f05a36a8f..fd7f50e2e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -365,6 +365,12 @@ stackhpc_pulp_rpm_repos: base_path: "opensearch-dashboards/2.x/yum/" required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + - name: DOCA Online Repo {{ stackhpc_pulp_doca_host_version }} + url: "{{ stackhpc_release_pulp_content_url }}/doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/{{ stackhpc_pulp_repo_doca_version }}" + distribution_name: "doca-" + base_path: "doca/{{ stackhpc_pulp_doca_host_version }}/rhel9.4/x86_64/" + required: "{{ stackhpc_pulp_sync_el_9 }}" + # RPM repositories stackhpc_pulp_repository_rpm_repos: >- {%- set rpm_repos = [] -%} @@ -380,6 +386,7 @@ stackhpc_pulp_repository_rpm_repos: >- {%- endif -%} {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(rpm_repo)) -%} {%- endfor -%} + {%- set _ = rpm_repos.append(stackhpc_pulp_rpm_repos_default_config | combine(stackhpc_pulp_repo_doca_ofed_rhel9)) -%} {{ rpm_repos }} # Publication format is a subset of distribution. diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 8526eb7cd..504fff3e2 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -52,6 +52,10 @@ stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_repo_distribution }}" ############################################################################### # RPMs +# OFED +stackhpc_repo_ofed_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/doca_ofed/{ stackhpc_pulp_doca_ofed_version }}/rhel9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}/x86_64/{{ stackhpc_repo_rocky_9_doca_ofed_version }}" +stackhpc_repo_rocky_9_doca_ofed_version: "{{ stackhpc_repo_distribution }}" + # Grafana stackhpc_repo_grafana_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/grafana/oss/rpm/{{ stackhpc_repo_grafana_version }}" stackhpc_repo_grafana_version: "{{ stackhpc_repo_distribution }}"