diff --git a/containerfiles/rocky-latest-doca-ofed b/containerfiles/rocky-latest-doca-ofed new file mode 100644 index 0000000..60a3f12 --- /dev/null +++ b/containerfiles/rocky-latest-doca-ofed @@ -0,0 +1,60 @@ +# Based on https://github.com/openstack/diskimage-builder/blob/master/diskimage_builder/elements/rocky-container/containerfiles/9 + +FROM quay.io/rockylinux/rockylinux:9 + +# Workaround for containerized kernel tools +RUN echo "9.6" > /etc/dnf/vars/releasefullver && \ +# Can't update packages in DIB once OFED is installed as it provides some "older" packages +RUN dnf update -y \ + ; dnf -y install chkconfig + +RUN dnf group install -y 'Minimal Install' --allowerasing && \ + dnf install -y findutils util-linux sudo python3 NetworkManager \ + NetworkManager-config-server curl libcurl --allowerasing + +RUN systemctl unmask console-getty.service dev-hugepages.mount \ + getty.target sys-fs-fuse-connections.mount systemd-logind.service \ + systemd-remount-fs.service + +# /etc/machine-id needs to be populated for /bin/kernel-install to +# correctly copy kernels into /boot. We will clear this out in the +# final image. +RUN systemd-machine-id-setup + +# Install base build dependencies required by DOCA OFED +RUN dnf config-manager --set-enabled appstream crb && \ + dnf install -y \ + perl cmake kernel-devel \ + python3-devel perl-generators python3 rpm-build \ + elfutils-libelf-devel zlib-devel gcc-c++ gdb-headless \ + glib2-devel patch lsof libmnl openssl-devel \ + pciutils-devel pkgconf-pkg-config libstdc++-devel \ + libnl3-devel libtool numactl-devel systemd-devel \ + kernel-rpm-macros glibc-devel pciutils gcc valgrind-devel \ + iptables-devel bison libdb-devel elfutils-devel \ + tcsh binutils-devel flex gcc-gfortran python3-Cython \ + python3-docutils libmnl-devel wget curl + +# Install DOCA OFED 2.9.3 and use doca-kernel-support +# Set DOCA version +ENV DOCA_VERSION=2.9.3 + +# Create repository file (multiline, because parser issues) +RUN echo "[doca]" > /etc/yum.repos.d/doca.repo && \ + echo "name=DOCA Online Repo" >> /etc/yum.repos.d/doca.repo && \ + echo "baseurl=https://linux.mellanox.com/public/repo/doca/${DOCA_VERSION}/rhel9.6/x86_64/" >> /etc/yum.repos.d/doca.repo && \ + echo "enabled=1" >> /etc/yum.repos.d/doca.repo && \ + echo "gpgcheck=0" >> /etc/yum.repos.d/doca.repo + +# Install packages (query kernel version for doca-kernel-support using rpm, as uname -r returns builder kernel version) +RUN dnf makecache && \ + dnf install -y doca-ofed doca-extra && \ + LATEST_KERNEL=$(rpm -qa kernel-core --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}\n' | sort -V | tail -n 1) && \ + /opt/mellanox/doca/tools/doca-kernel-support -k "${LATEST_KERNEL}" && \ + GENERATED_RPM=$(find /tmp -name 'doca-kernel-repo-*.rpm' | head -n 1) && \ + echo "Installing kernel repo from: ${GENERATED_RPM}" && \ + rpm -i "${GENERATED_RPM}" && \ + dnf makecache && \ + dnf install -y doca-ofed-userspace && \ + dnf install -y --disablerepo=doca doca-kernel-* && \ + dnf clean all \ No newline at end of file diff --git a/examples/images.yml b/examples/images.yml index dbacc0f..9f14031 100644 --- a/examples/images.yml +++ b/examples/images.yml @@ -19,6 +19,7 @@ openstack_images: - "{{ openstack_image_cirros_0_6_0 }}" - "{{ openstack_image_rocky9 }}" - "{{ openstack_image_ubuntu_jammy }}" + - "{{ openstack_image_rocky9_doca_ofed }}" # Common GRUB settings for VM images openstack_grub_env_common: @@ -129,3 +130,210 @@ openstack_image_ubuntu_jammy: env: DIB_RELEASE: "jammy" DIB_CLOUD_INIT_DATASOURCES: "ConfigDrive" + +# Rocky Linux 9 with Doca Ofed enabled. +openstack_image_rocky9_doca_ofed: + name: "Rocky9-doca-ofed" + type: raw + elements: + - "rocky-container" + - "cloud-init" + - "cloud-init-datasources" + - "enable-serial-console" + - "block-device-efi" + - "vm" + - "openssh-server" + - "dracut-regenerate" + visibility: "public" + packages: + - "git" + - "tmux" + - "vim-enhanced" + - "lshw" + - "pciutils" + - "infiniband-diags" + - "ethtool" + - "less" + - "logrotate" + - "net-tools" + - "nvme-cli" + - "python3" + - "smartmontools" + - "NetworkManager-config-server" + - "linux-firmware" + - "cloud-utils-growpart" + env: + DIB_AVOID_PACKAGES_UPDATE: 1 + DIB_BLOCK_DEVICE_CONFIG: "{{ stackhpc_dib_block_device_config_uefi_lvm }}" + DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_dib_dracut_enabled_modules_default_config }}" + DIB_BOOTLOADER_DEFAULT_CMDLINE: "nofb nomodeset gfxpayload=text net.ifnames=1 rd.auto" + DIB_GRUB_TIMEOUT: "5" + DIB_GRUB_TIMEOUT_STYLE: "menu" + DIB_CONTAINERFILE_DOCKERFILE: "{{ playbook_dir }}/../containerfiles/rocky-latest-doca-ofed" + DIB_CONTAINERFILE_NETWORK_DRIVER: host + DIB_CONTAINERFILE_RUNTIME: docker + YUM: dnf + DIB_CLOUD_INIT_DATASOURCES: "OpenStack, ConfigDrive" + DIB_RELEASE: "9.6" + # Workaround for stack user home ownership bug + DIB_IMAGE_CACHE: "/tmp/yum" + DIB_SUDOERS_FILENAME: "no-fqdn" + # Avoid DNS queries during sudo commands, since we might not always have working DNS. + DIB_SUDOERS_CONFIG: | + Defaults !fqdn + +# StackHPC overcloud DIB image block device configuration. +# This image layout conforms to the CIS partition benchmarks. +# This configuration builds a UEFI-compatible image with 3 partitions. +# * p0: EFI ESP bootloader +# * p1: EFI BSP +# * p2: LVM PV (rootpv) +# The rootpv PV is in the rootvg VG, and has the following LVs: +# * lv_root -> / +# * lv_tmp -> /tmp +# * lv_var -> /var +# * lv_var_tmp -> /var/tmp +# * lv_log -> /var/log +# * lv_audit -> /var/log/audit +# * lv_home -> /home + +stackhpc_dib_block_device_config_uefi_lvm: | + - local_loop: + name: image0 + size: 20GiB + - partitioning: + base: image0 + label: gpt + partitions: + - name: ESP + type: 'EF00' + size: 500MiB + mkfs: + type: vfat + mount: + mount_point: /boot/efi + fstab: + options: "defaults" + fsck-passno: 2 + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + type: '8E00' + flags: [ boot ] + size: 100% + - lvm: + name: lvm + base: [ root ] + pvs: + - name: rootpv + base: root + options: [ "--force" ] + vgs: + - name: rootvg + base: [ "rootpv" ] + options: [ "--force" ] + lvs: + - name: lv_root + base: rootvg + size: 5G + - name: lv_tmp + base: rootvg + size: 1G + - name: lv_var + base: rootvg + size: 1G + - name: lv_var_tmp + base: rootvg + size: 1G + - name: lv_log + base: rootvg + size: 1G + - name: lv_audit + base: rootvg + size: 128M + - name: lv_home + base: rootvg + size: 128M + - mkfs: + name: fs_root + base: lv_root + type: ext4 + label: "rootfs" + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 + - mkfs: + name: fs_tmp + base: lv_tmp + type: ext4 + label: "tmpfs" + mount: + mount_point: /tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_var + base: lv_var + type: ext4 + label: "varfs" + mount: + mount_point: /var + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_var_tmp + base: lv_var_tmp + type: ext4 + label: "vartmpfs" + mount: + mount_point: /var/tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_log + base: lv_log + type: ext4 + label: "logfs" + mount: + mount_point: /var/log + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_audit + base: lv_audit + type: ext4 + label: "auditfs" + mount: + mount_point: /var/log/audit + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_home + base: lv_home + type: ext4 + label: "homefs" + mount: + mount_point: /home + fstab: + options: "rw,nodev" + fsck-passno: 2 + +# StackHPC overcloud DIB image Dracut module configuration. +stackhpc_dib_dracut_enabled_modules_default_config: | + - name: crypt + packages: + - cryptsetup + - name: lvm + packages: + - lvm2 + - name: mdraid + packages: + - mdraid