From 6f6b57c7743f6f0d1ec28e38c58049370e015703 Mon Sep 17 00:00:00 2001 From: Manimaran-MM Date: Wed, 14 Jan 2026 13:24:17 +0530 Subject: [PATCH 1/3] fix: pick the kernel version compatible to gpfs based on global search instead of hardcoding Current Issue: - CentOS has rolling builds and it requires manual intervention to identify and hardcode the builds - It shows as test failure which may lead to miss issues Proposed Solution: - To have a generic mechanism to identify the kernel version for the qcow image before loading - Provide the constraint based on GPFS versions - Parse, compare and pick the version that is satisfying the kernel version Signed-off-by: Manimaran-MM --- ci_utils/common/helpers.py | 81 +++++++++++++++++++++++++++- ci_utils/gpfs/gpfs_setup.py | 24 +++++++-- ci_utils/virtual_machine/vm_setup.py | 4 +- tests/test_pynfs_cthon.py | 8 ++- 4 files changed, 108 insertions(+), 9 deletions(-) diff --git a/ci_utils/common/helpers.py b/ci_utils/common/helpers.py index 46549d8c..e3578661 100644 --- a/ci_utils/common/helpers.py +++ b/ci_utils/common/helpers.py @@ -1,7 +1,9 @@ import os import json +import re import subprocess from typing import Dict, Any +from distutils.version import LooseVersion from ci_utils.common.logger import get_logger @@ -134,4 +136,81 @@ def run_cmd(session, cmd, check=True, timeout=3600, source_bashrc=False): logger.error(f"Failure Output: {out}") raise RuntimeError(err) logger.info(f"[REMOTE] Command output for {cmd_to_run} with return code {code}: Output: {out.strip()} \n {err.strip()}\n") - return out.strip(), code \ No newline at end of file + return out.strip(), code + +# ----------------------- +# Identify the matching Qcow image from the list of available images +# ----------------------- +def identify_matching_qcow_image(session,centos_arch, centos_version, image_base_url, backend_type="gpfs", version_constraints=None): + """ + Identify the matching Qcow image from the list of available images + Args: + centos_arch (str): CentOS architecture (x86_64, aarch64) + centos_version (str): CentOS version (9, 8, 7) + image_base_url (str): Base URL of the image list + backend_type (str): Backend type (gpfs, pjdfs) + Returns: + str: Matching Qcow image name + """ + logger.info(f"[STEP]: Identifying matching Qcow image for CentOS {centos_version} {centos_arch} {backend_type} with version constraints {version_constraints}") + run_cmd(session, "systemctl enable --now libvirtd") + run_cmd(session, "dnf install -y libguestfs-tools-c") + available_images = run_cmd( + session, + rf"curl -s {image_base_url} | grep -oP 'CentOS-Stream-GenericCloud-{centos_arch}-{centos_version}.*?\.qcow2(?=\")'" + ) + + logger.info(f"[INFO] Available images: {available_images}") + + # Parse available_images output into a list (assuming it's a string with newlines) + # Note: available_images is a tuple (output, code) from run_cmd, so we need to extract the output + images_list = available_images[0].strip().split('\n') if isinstance(available_images, tuple) else available_images.strip().split('\n') + images_list = [img for img in images_list if img.strip()] # Remove empty strings + + # Create the destination directory if it doesn't exist + run_cmd(session, "mkdir -p /var/lib/libguestfs/images/") + + # Iterate from the last image backwards + for image_name in reversed(images_list): + image_name = image_name.strip() + if not image_name: + continue + + logger.info(f"Processing image: {image_name}") + + # wget operation with baseurl + image name + image_url = f"{image_base_url.rstrip('/')}/{image_name}" + run_cmd(session, f"wget -q {image_url}") + + # Move the downloaded image to /var/lib/libguestfs/images/ + run_cmd(session, f"mv {image_name} /var/lib/libguestfs/images/") + + # chmod 644 on all qcow2 images in the directory + run_cmd(session, "chmod 644 /var/lib/libguestfs/images/*.qcow2") + + # Check kernel version using guestfish + image_path = f"/var/lib/libguestfs/images/{image_name}" + kernel_version, _ = run_cmd(session, f'virt-ls -a "{image_path}" /usr/lib/modules/') + logger.info(f"[INFO] Kernel version for {image_name}: {kernel_version}") + + if version_constraints: + + kernel_ver_str = kernel_version[0].strip() if isinstance(kernel_version, tuple) else kernel_version.strip() + + # Normalize both versions + kernel_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", kernel_ver_str).group(1) + constraint_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", version_constraints).group(1) + + logger.info(f"[INFO] Comparing kernel {kernel_ver_str} (normalized: {kernel_normalized}) with constraint {version_constraints} (normalized: {constraint_normalized})") + + # Use LooseVersion for comparison + if LooseVersion(kernel_normalized) <= LooseVersion(constraint_normalized): + logger.info(f"[INFO] Kernel version meets constraint. Returning image URL: {image_url}") + return image_url + else: + logger.info(f"[INFO] Kernel version does NOT meet constraint, trying next image") + run_cmd(session, f"rm -f {image_path}") + continue + else: + logger.info(f"[INFO] No version constraints specified, returning image URL: {image_url}") + return image_url \ No newline at end of file diff --git a/ci_utils/gpfs/gpfs_setup.py b/ci_utils/gpfs/gpfs_setup.py index f1e8f510..b7accfd7 100644 --- a/ci_utils/gpfs/gpfs_setup.py +++ b/ci_utils/gpfs/gpfs_setup.py @@ -119,10 +119,26 @@ def _install_on_node(node, sess): run_cmd(sess, "dnf install -y kernel-devel-$(uname -r) kernel-headers-$(uname -r)") else: logger.info(f"Attempting Koji fetch...") - run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-devel-5.14.0-639.el9.x86_64.rpm") - run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-headers-5.14.0-639.el9.x86_64.rpm") - run_cmd(sess, "ls -la") - run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-639.el9.x86_64.rpm ./kernel-headers-5.14.0-639.el9.x86_64.rpm") + # Get kernel version and extract components for Koji URL + # Example: 5.14.0-658.el9.x86_64 -> major=5.14.0, build=658, release=el9 + kernel_ver, _ = run_cmd(sess, "uname -r") + kernel_ver = kernel_ver.strip() + # Parse: 5.14.0-658.el9.x86_64 -> extract 5.14.0, 658, el9 + match = re.match(r'(\d+\.\d+\.\d+)-(\d+)\.(el\d+)', kernel_ver) + if match: + major, build, release = match.groups() + koji_base = f"https://kojihub.stream.centos.org/kojifiles/packages/kernel/{major}/{build}.{release}/x86_64" + kernel_pkg_ver = f"{major}-{build}.{release}" + run_cmd(sess, f"wget {koji_base}/kernel-devel-{kernel_pkg_ver}.x86_64.rpm") + run_cmd(sess, f"wget {koji_base}/kernel-headers-{kernel_pkg_ver}.x86_64.rpm") + run_cmd(sess, "ls -la") + run_cmd(sess, f"dnf -y install openssl-fips-provider ./kernel-devel-{kernel_pkg_ver}.x86_64.rpm ./kernel-headers-{kernel_pkg_ver}.x86_64.rpm") + else: + logger.warning(f"Could not parse kernel version {kernel_ver}, falling back to default") + run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-devel-5.14.0-570.el9.x86_64.rpm") + run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-headers-5.14.0-570.el9.x86_64.rpm") + run_cmd(sess, "ls -la") + run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-570.el9.x86_64.rpm ./kernel-headers-5.14.0-570.el9.x86_64.rpm") run_cmd( sess, "yum -y install " diff --git a/ci_utils/virtual_machine/vm_setup.py b/ci_utils/virtual_machine/vm_setup.py index c674dc27..66017622 100644 --- a/ci_utils/virtual_machine/vm_setup.py +++ b/ci_utils/virtual_machine/vm_setup.py @@ -77,9 +77,9 @@ def setup_network(self): self.session, f"cat > {remote_xml} <<'EOF'\n{xml_content}\nEOF" ) - run_cmd(self.session, f"virsh net-define {remote_xml}") + run_cmd(self.session, f"virsh net-define {remote_xml}", check=False) run_cmd(self.session, "virsh net-start default") - run_cmd(self.session, "virsh net-autostart default") + run_cmd(self.session, "virsh net-autostart default" self.network = "default" # ----------------------- diff --git a/tests/test_pynfs_cthon.py b/tests/test_pynfs_cthon.py index f055021b..4a47d07b 100644 --- a/tests/test_pynfs_cthon.py +++ b/tests/test_pynfs_cthon.py @@ -367,6 +367,7 @@ def test_pynfs_gpfs(create_session, cmake_flags): vm_name = "centos9-vm" username = "root" ssh_key = "/root/.ssh/id_rsa.pub" + version_constraints = "5.14.0-570.62.1.el9_6" #Assuming GPFS 6.0 https://www.ibm.com/docs/en/STXKQY/gpfsclustersfaq.html#fsi gerrit_host = os.getenv("GERRIT_HOST", "review.gerrithub.io") gerrit_project = os.getenv("GERRIT_PROJECT", "ffilz/nfs-ganesha") gerrit_refspec = os.getenv("GERRIT_REFSPEC", "") @@ -386,13 +387,16 @@ def test_pynfs_gpfs(create_session, cmake_flags): # ----------------------- # VM Setup # ----------------------- + image_url = identify_matching_qcow_image(server_session, "x86_64", "9", "https://cloud.centos.org/centos/9-stream/x86_64/images/", version_constraints=version_constraints) + logger.info("Image URL: %s", image_url) + logger.info("Setting up VM on baremetal node: %s", server_node) vm = VMManager( session=server_session, workspace=server_workspace, vm_name=vm_name, - image_url="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2", - image_name="CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2", + image_url=image_url, + image_name=image_url.split("/")[-1], vm_cpu="2", vm_ram="8192", vm_disk="30G", From f7c2d150c4b30a2934af2bf46896d78ebf67bd1c Mon Sep 17 00:00:00 2001 From: Manimaran-MM Date: Wed, 14 Jan 2026 20:58:15 +0530 Subject: [PATCH 2/3] fix: vfs changes related to conf to made at the conf level - Based on the Enable ACL and Security Label param, update the conf beforehand rather than updating after bring up Signed-off-by: Manimaran-MM --- ci_utils/vfs/vfs_setup.py | 67 +++++++++++++++------------- ci_utils/virtual_machine/vm_setup.py | 2 +- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/ci_utils/vfs/vfs_setup.py b/ci_utils/vfs/vfs_setup.py index fabf1cfe..1fa812c5 100644 --- a/ci_utils/vfs/vfs_setup.py +++ b/ci_utils/vfs/vfs_setup.py @@ -47,11 +47,11 @@ def configure_export(self): Path = "/{self.vfs_volume}"; Pseudo = "/{self.vfs_volume}"; Access_type = RW; - Disable_ACL = True; + Disable_ACL = {str(not self.enable_acl).capitalize()}; Protocols = "3","4"; Transports = "UDP","TCP"; SecType = "sys"; - Security_Label = False; + Security_Label = {str(self.security_label).capitalize()}; FSAL {{ Name = VFS; }} @@ -78,41 +78,44 @@ def validate_export(self): run_cmd(self.session, "grep --with-filename -e '' /etc/ganesha/exports/*.conf", check=False) raise RuntimeError(f"Export {self.vfs_volume} not found!") + # Commenting out below enablement as the changes are already done in the export configuration + # Retaining the code for future reference for enabling other features + # ------------------------------- # Enable ACL if required # ------------------------------- - def enable_acl_if_required(self): - logger.info("[TEST]: Checking if ACL needs to be enabled") - if self.enable_acl: - logger.info("Enabling ACL for volume...") - run_cmd(self.session, f"sed -i s/'Disable_ACL = .*'/'Disable_ACL = false;'/g {self.export_conf}") - run_cmd(self.session, f"cat {self.export_conf}") - export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'") - run_cmd( - self.session, - f"dbus-send --type=method_call --print-reply --system " - f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr " - f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} " - f"string:\"EXPORT(Export_Id = {export_id})\"" - ) + # def enable_acl_if_required(self): + # logger.info("[TEST]: Checking if ACL needs to be enabled") + # if self.enable_acl: + # # logger.info("Enabling ACL for volume...") + # # run_cmd(self.session, f"sed -i s/'Disable_ACL = .*'/'Disable_ACL = false;'/g {self.export_conf}") + # run_cmd(self.session, f"cat {self.export_conf}") + # export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'") + # run_cmd( + # self.session, + # f"dbus-send --type=method_call --print-reply --system " + # f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr " + # f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} " + # f"string:\"EXPORT(Export_Id = {export_id})\"" + # ) # ------------------------------- # Enable Security_Label if required # ------------------------------- - def enable_security_label_if_required(self): - logger.info("[TEST]: Checking if Security_Label needs to be enabled") - if self.security_label: - logger.info("Enabling Security_Label for volume...") - run_cmd(self.session, f"sed -i s/'Security_Label = .*'/'Security_Label = True;'/g {self.export_conf}") - run_cmd(self.session, f"cat {self.export_conf}") - export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'") - run_cmd( - self.session, - f"dbus-send --type=method_call --print-reply --system " - f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr " - f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} " - f"string:\"EXPORT(Export_Id = {export_id})\"" - ) + # def enable_security_label_if_required(self): + # logger.info("[TEST]: Checking if Security_Label needs to be enabled") + # if self.security_label: + # logger.info("Enabling Security_Label for volume...") + # run_cmd(self.session, f"sed -i s/'Security_Label = .*'/'Security_Label = True;'/g {self.export_conf}") + # run_cmd(self.session, f"cat {self.export_conf}") + # export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'") + # run_cmd( + # self.session, + # f"dbus-send --type=method_call --print-reply --system " + # f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr " + # f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} " + # f"string:\"EXPORT(Export_Id = {export_id})\"" + # ) # ------------------------------- # Main export workflow @@ -123,6 +126,6 @@ def export_volume(self): self.configure_export() run_cmd(self.session, "sleep 5") self.validate_export() - self.enable_acl_if_required() - self.enable_security_label_if_required() + # self.enable_acl_if_required() + # self.enable_security_label_if_required() logger.info("Export completed successfully.") diff --git a/ci_utils/virtual_machine/vm_setup.py b/ci_utils/virtual_machine/vm_setup.py index 66017622..e9ac0890 100644 --- a/ci_utils/virtual_machine/vm_setup.py +++ b/ci_utils/virtual_machine/vm_setup.py @@ -79,7 +79,7 @@ def setup_network(self): ) run_cmd(self.session, f"virsh net-define {remote_xml}", check=False) run_cmd(self.session, "virsh net-start default") - run_cmd(self.session, "virsh net-autostart default" + run_cmd(self.session, "virsh net-autostart default") self.network = "default" # ----------------------- From 788129852f5d8041b87b304e09016afe017d2ea8 Mon Sep 17 00:00:00 2001 From: Manimaran-MM Date: Fri, 16 Jan 2026 11:10:07 +0530 Subject: [PATCH 3/3] update: Update kernel version according to gpfs Signed-off-by: Manimaran-MM --- ci_utils/vfs/vfs_setup.py | 2 +- tests/test_pynfs_cthon.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci_utils/vfs/vfs_setup.py b/ci_utils/vfs/vfs_setup.py index 1fa812c5..5912a4de 100644 --- a/ci_utils/vfs/vfs_setup.py +++ b/ci_utils/vfs/vfs_setup.py @@ -80,7 +80,7 @@ def validate_export(self): # Commenting out below enablement as the changes are already done in the export configuration # Retaining the code for future reference for enabling other features - + # ------------------------------- # Enable ACL if required # ------------------------------- diff --git a/tests/test_pynfs_cthon.py b/tests/test_pynfs_cthon.py index 4a47d07b..9e53d7c8 100644 --- a/tests/test_pynfs_cthon.py +++ b/tests/test_pynfs_cthon.py @@ -367,7 +367,7 @@ def test_pynfs_gpfs(create_session, cmake_flags): vm_name = "centos9-vm" username = "root" ssh_key = "/root/.ssh/id_rsa.pub" - version_constraints = "5.14.0-570.62.1.el9_6" #Assuming GPFS 6.0 https://www.ibm.com/docs/en/STXKQY/gpfsclustersfaq.html#fsi + version_constraints = "5.14.0-611.16.1.el9_7" #Assuming GPFS 6.0 https://www.ibm.com/docs/en/STXKQY/gpfsclustersfaq.html#fsi gerrit_host = os.getenv("GERRIT_HOST", "review.gerrithub.io") gerrit_project = os.getenv("GERRIT_PROJECT", "ffilz/nfs-ganesha") gerrit_refspec = os.getenv("GERRIT_REFSPEC", "") @@ -561,4 +561,4 @@ def test_pynfs_gpfs(create_session, cmake_flags): with open(SUMMARY_FILE, "a", encoding="utf-8") as f: f.write(failure_msg) with open(SUMMARY_STATUS, "a", encoding="utf-8") as f: - f.write("\nFailed") \ No newline at end of file + f.write("\nFailed")