fix: pick the kernel version compatible to gpfs based on global search

ci_utils/common/helpers.py

@@ -1,7 +1,9 @@
 import os
 import json
+import re
 import subprocess
 from typing import Dict, Any
+from distutils.version import LooseVersion
 
 from ci_utils.common.logger import get_logger
 
@@ -134,4 +136,81 @@ def run_cmd(session, cmd, check=True, timeout=3600, source_bashrc=False):
         logger.error(f"Failure Output: {out}")
         raise RuntimeError(err)
     logger.info(f"[REMOTE] Command output for {cmd_to_run} with return code {code}: Output: {out.strip()} \n {err.strip()}\n")
-    return out.strip(), code
+    return out.strip(), code
+
+# -----------------------
+# Identify the matching Qcow image from the list of available images
+# -----------------------
+def identify_matching_qcow_image(session,centos_arch, centos_version, image_base_url, backend_type="gpfs", version_constraints=None):
+    """
+    Identify the matching Qcow image from the list of available images
+    Args:
+        centos_arch (str): CentOS architecture (x86_64, aarch64)
+        centos_version (str): CentOS version (9, 8, 7)
+        image_base_url (str): Base URL of the image list
+        backend_type (str): Backend type (gpfs, pjdfs)
+    Returns:
+        str: Matching Qcow image name
+    """
+    logger.info(f"[STEP]: Identifying matching Qcow image for CentOS {centos_version} {centos_arch} {backend_type} with version constraints {version_constraints}")
+    run_cmd(session, "systemctl enable --now libvirtd")
+    run_cmd(session, "dnf install -y libguestfs-tools-c")
+    available_images = run_cmd(
+        session,
+        rf"curl -s {image_base_url} | grep -oP 'CentOS-Stream-GenericCloud-{centos_arch}-{centos_version}.*?\.qcow2(?=\")'"
+    )
+
+    logger.info(f"[INFO] Available images: {available_images}")
+
+    # Parse available_images output into a list (assuming it's a string with newlines)
+    # Note: available_images is a tuple (output, code) from run_cmd, so we need to extract the output
+    images_list = available_images[0].strip().split('\n') if isinstance(available_images, tuple) else available_images.strip().split('\n')
+    images_list = [img for img in images_list if img.strip()]  # Remove empty strings
+
+    # Create the destination directory if it doesn't exist
+    run_cmd(session, "mkdir -p /var/lib/libguestfs/images/")
+
+    # Iterate from the last image backwards
+    for image_name in reversed(images_list):
+        image_name = image_name.strip()
+        if not image_name:
+            continue
+
+        logger.info(f"Processing image: {image_name}")
+
+        # wget operation with baseurl + image name
+        image_url = f"{image_base_url.rstrip('/')}/{image_name}"
+        run_cmd(session, f"wget -q {image_url}")
+
+        # Move the downloaded image to /var/lib/libguestfs/images/
+        run_cmd(session, f"mv {image_name} /var/lib/libguestfs/images/")
+
+        # chmod 644 on all qcow2 images in the directory
+        run_cmd(session, "chmod 644 /var/lib/libguestfs/images/*.qcow2")
+
+        # Check kernel version using guestfish
+        image_path = f"/var/lib/libguestfs/images/{image_name}"
+        kernel_version, _ = run_cmd(session, f'virt-ls -a "{image_path}" /usr/lib/modules/')
+        logger.info(f"[INFO] Kernel version for {image_name}: {kernel_version}")
+
+        if version_constraints:
+
+            kernel_ver_str = kernel_version[0].strip() if isinstance(kernel_version, tuple) else kernel_version.strip()
+
+            # Normalize both versions
+            kernel_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", kernel_ver_str).group(1)
+            constraint_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", version_constraints).group(1)
+
+            logger.info(f"[INFO] Comparing kernel {kernel_ver_str} (normalized: {kernel_normalized}) with constraint {version_constraints} (normalized: {constraint_normalized})")
+
+            # Use LooseVersion for comparison
+            if LooseVersion(kernel_normalized) <= LooseVersion(constraint_normalized):
+                logger.info(f"[INFO] Kernel version meets constraint. Returning image URL: {image_url}")
+                return image_url
+            else:
+                logger.info(f"[INFO] Kernel version does NOT meet constraint, trying next image")
+                run_cmd(session, f"rm -f {image_path}")
+                continue
+        else:
+            logger.info(f"[INFO] No version constraints specified, returning image URL: {image_url}")
+            return image_url

ci_utils/gpfs/gpfs_setup.py

@@ -119,10 +119,26 @@ def _install_on_node(node, sess):
                 run_cmd(sess, "dnf install -y kernel-devel-$(uname -r) kernel-headers-$(uname -r)")
             else:
                 logger.info(f"Attempting Koji fetch...")
-                run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-devel-5.14.0-639.el9.x86_64.rpm")
-                run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-headers-5.14.0-639.el9.x86_64.rpm")
-                run_cmd(sess, "ls -la")
-                run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-639.el9.x86_64.rpm ./kernel-headers-5.14.0-639.el9.x86_64.rpm")
+                # Get kernel version and extract components for Koji URL
+                # Example: 5.14.0-658.el9.x86_64 -> major=5.14.0, build=658, release=el9
+                kernel_ver, _ = run_cmd(sess, "uname -r")
+                kernel_ver = kernel_ver.strip()
+                # Parse: 5.14.0-658.el9.x86_64 -> extract 5.14.0, 658, el9
+                match = re.match(r'(\d+\.\d+\.\d+)-(\d+)\.(el\d+)', kernel_ver)
+                if match:
+                    major, build, release = match.groups()
+                    koji_base = f"https://kojihub.stream.centos.org/kojifiles/packages/kernel/{major}/{build}.{release}/x86_64"
+                    kernel_pkg_ver = f"{major}-{build}.{release}"
+                    run_cmd(sess, f"wget {koji_base}/kernel-devel-{kernel_pkg_ver}.x86_64.rpm")
+                    run_cmd(sess, f"wget {koji_base}/kernel-headers-{kernel_pkg_ver}.x86_64.rpm")
+                    run_cmd(sess, "ls -la")
+                    run_cmd(sess, f"dnf -y install openssl-fips-provider ./kernel-devel-{kernel_pkg_ver}.x86_64.rpm ./kernel-headers-{kernel_pkg_ver}.x86_64.rpm")
+                else:
+                    logger.warning(f"Could not parse kernel version {kernel_ver}, falling back to default")
+                    run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-devel-5.14.0-570.el9.x86_64.rpm")
+                    run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-headers-5.14.0-570.el9.x86_64.rpm")
+                    run_cmd(sess, "ls -la")
+                    run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-570.el9.x86_64.rpm ./kernel-headers-5.14.0-570.el9.x86_64.rpm")
             run_cmd(
                 sess,
                 "yum -y install "

ci_utils/vfs/vfs_setup.py

@@ -47,11 +47,11 @@ def configure_export(self):
     Path = "/{self.vfs_volume}";
     Pseudo = "/{self.vfs_volume}";
     Access_type = RW;
-    Disable_ACL = True;
+    Disable_ACL = {str(not self.enable_acl).capitalize()};
     Protocols = "3","4";
     Transports = "UDP","TCP";
     SecType = "sys";
-    Security_Label = False;
+    Security_Label = {str(self.security_label).capitalize()};
     FSAL {{
         Name = VFS;
     }}
@@ -78,41 +78,44 @@ def validate_export(self):
             run_cmd(self.session, "grep --with-filename -e '' /etc/ganesha/exports/*.conf", check=False)
             raise RuntimeError(f"Export {self.vfs_volume} not found!")
 
+    # Commenting out below enablement as the changes are already done in the export configuration
+    # Retaining the code for future reference for enabling other features
+
     # -------------------------------
     # Enable ACL if required
     # -------------------------------
-    def enable_acl_if_required(self):
-        logger.info("[TEST]: Checking if ACL needs to be enabled")
-        if self.enable_acl:
-            logger.info("Enabling ACL for volume...")
-            run_cmd(self.session, f"sed -i s/'Disable_ACL = .*'/'Disable_ACL = false;'/g {self.export_conf}")
-            run_cmd(self.session, f"cat {self.export_conf}")
-            export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'")
-            run_cmd(
-                self.session,
-                f"dbus-send --type=method_call --print-reply --system "
-                f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr "
-                f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} "
-                f"string:\"EXPORT(Export_Id = {export_id})\""
-            )
+    # def enable_acl_if_required(self):
+        # logger.info("[TEST]: Checking if ACL needs to be enabled")
+        # if self.enable_acl:
+        #     # logger.info("Enabling ACL for volume...")
+        #     # run_cmd(self.session, f"sed -i s/'Disable_ACL = .*'/'Disable_ACL = false;'/g {self.export_conf}")
+        #     run_cmd(self.session, f"cat {self.export_conf}")
+        #     export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'")
+        #     run_cmd(
+        #         self.session,
+        #         f"dbus-send --type=method_call --print-reply --system "
+        #         f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr "
+        #         f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} "
+        #         f"string:\"EXPORT(Export_Id = {export_id})\""
+        #     )
 
     # -------------------------------
     # Enable Security_Label if required
     # -------------------------------
-    def enable_security_label_if_required(self):
-        logger.info("[TEST]: Checking if Security_Label needs to be enabled")
-        if self.security_label:
-            logger.info("Enabling Security_Label for volume...")
-            run_cmd(self.session, f"sed -i s/'Security_Label = .*'/'Security_Label = True;'/g {self.export_conf}")
-            run_cmd(self.session, f"cat {self.export_conf}")
-            export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'")
-            run_cmd(
-                self.session,
-                f"dbus-send --type=method_call --print-reply --system "
-                f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr "
-                f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} "
-                f"string:\"EXPORT(Export_Id = {export_id})\""
-            )
+    # def enable_security_label_if_required(self):
+    #     logger.info("[TEST]: Checking if Security_Label needs to be enabled")
+    #     if self.security_label:
+    #         logger.info("Enabling Security_Label for volume...")
+    #         run_cmd(self.session, f"sed -i s/'Security_Label = .*'/'Security_Label = True;'/g {self.export_conf}")
+    #         run_cmd(self.session, f"cat {self.export_conf}")
+    #         export_id, _ = run_cmd(self.session, f"grep 'Export_Id' {self.export_conf} | sed 's/^[[:space:]]*Export_Id.*=[[:space:]]*\\([0-9]*\\).*/\\1/'")
+    #         run_cmd(
+    #             self.session,
+    #             f"dbus-send --type=method_call --print-reply --system "
+    #             f"--dest=org.ganesha.nfsd /org/ganesha/nfsd/ExportMgr "
+    #             f"org.ganesha.nfsd.exportmgr.UpdateExport string:{self.export_conf} "
+    #             f"string:\"EXPORT(Export_Id = {export_id})\""
+    #         )
 
     # -------------------------------
     # Main export workflow
@@ -123,6 +126,6 @@ def export_volume(self):
         self.configure_export()
         run_cmd(self.session, "sleep 5")
         self.validate_export()
-        self.enable_acl_if_required()
-        self.enable_security_label_if_required()
+        # self.enable_acl_if_required()
+        # self.enable_security_label_if_required()
         logger.info("Export completed successfully.")

ci_utils/virtual_machine/vm_setup.py

@@ -77,7 +77,7 @@ def setup_network(self):
                 self.session,
                 f"cat > {remote_xml} <<'EOF'\n{xml_content}\nEOF"
             )
-            run_cmd(self.session, f"virsh net-define {remote_xml}")
+            run_cmd(self.session, f"virsh net-define {remote_xml}", check=False)
             run_cmd(self.session, "virsh net-start default")
             run_cmd(self.session, "virsh net-autostart default")
             self.network = "default"

tests/test_pynfs_cthon.py

@@ -367,6 +367,7 @@ def test_pynfs_gpfs(create_session, cmake_flags):
         vm_name = "centos9-vm"
         username = "root"
         ssh_key = "/root/.ssh/id_rsa.pub"
+        version_constraints = "5.14.0-611.16.1.el9_7" #Assuming GPFS 6.0 https://www.ibm.com/docs/en/STXKQY/gpfsclustersfaq.html#fsi
         gerrit_host = os.getenv("GERRIT_HOST", "review.gerrithub.io")
         gerrit_project = os.getenv("GERRIT_PROJECT", "ffilz/nfs-ganesha")
         gerrit_refspec = os.getenv("GERRIT_REFSPEC", "")
@@ -386,13 +387,16 @@ def test_pynfs_gpfs(create_session, cmake_flags):
         # -----------------------
         # VM Setup
         # -----------------------
+        image_url = identify_matching_qcow_image(server_session, "x86_64", "9", "https://cloud.centos.org/centos/9-stream/x86_64/images/", version_constraints=version_constraints)
+        logger.info("Image URL: %s", image_url)
+
         logger.info("Setting up VM on baremetal node: %s", server_node)
         vm = VMManager(
             session=server_session,
             workspace=server_workspace,
             vm_name=vm_name,
-            image_url="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2",
-            image_name="CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2",
+            image_url=image_url,
+            image_name=image_url.split("/")[-1],
             vm_cpu="2",
             vm_ram="8192",
             vm_disk="30G",
@@ -557,4 +561,4 @@ def test_pynfs_gpfs(create_session, cmake_flags):
         with open(SUMMARY_FILE, "a", encoding="utf-8") as f:
             f.write(failure_msg)
         with open(SUMMARY_STATUS, "a", encoding="utf-8") as f:
-            f.write("\nFailed")
+            f.write("\nFailed")