fix: pick the kernel version compatible to gpfs based on global search instead of hardcoding

Current Issue:
- CentOS has rolling builds and it requires manual intervention to identify and hardcode the builds
- It shows as test failure which may lead to miss issues

Proposed Solution:
- To have a generic mechanism to identify the kernel version for the qcow image before loading
- Provide the constraint based on GPFS versions
- Parse, compare and pick the version that is satisfying the kernel version

Signed-off-by: Manimaran-MM <manim@redhat.com>

Author: Manimaran-MM

ci_utils/common/helpers.py

@@ -1,7 +1,9 @@
 import os
 import json
+import re
 import subprocess
 from typing import Dict, Any
+from distutils.version import LooseVersion
 
 from ci_utils.common.logger import get_logger
 
@@ -134,4 +136,81 @@ def run_cmd(session, cmd, check=True, timeout=3600, source_bashrc=False):
         logger.error(f"Failure Output: {out}")
         raise RuntimeError(err)
     logger.info(f"[REMOTE] Command output for {cmd_to_run} with return code {code}: Output: {out.strip()} \n {err.strip()}\n")
-    return out.strip(), code
+    return out.strip(), code
+
+# -----------------------
+# Identify the matching Qcow image from the list of available images
+# -----------------------
+def identify_matching_qcow_image(session,centos_arch, centos_version, image_base_url, backend_type="gpfs", version_constraints=None):
+    """
+    Identify the matching Qcow image from the list of available images
+    Args:
+        centos_arch (str): CentOS architecture (x86_64, aarch64)
+        centos_version (str): CentOS version (9, 8, 7)
+        image_base_url (str): Base URL of the image list
+        backend_type (str): Backend type (gpfs, pjdfs)
+    Returns:
+        str: Matching Qcow image name
+    """
+    logger.info(f"[STEP]: Identifying matching Qcow image for CentOS {centos_version} {centos_arch} {backend_type} with version constraints {version_constraints}")
+    run_cmd(session, "systemctl enable --now libvirtd")
+    run_cmd(session, "dnf install -y libguestfs-tools-c")
+    available_images = run_cmd(
+        session,
+        rf"curl -s {image_base_url} | grep -oP 'CentOS-Stream-GenericCloud-{centos_arch}-{centos_version}.*?\.qcow2(?=\")'"
+    )
+
+    logger.info(f"[INFO] Available images: {available_images}")
+  
+    # Parse available_images output into a list (assuming it's a string with newlines)
+    # Note: available_images is a tuple (output, code) from run_cmd, so we need to extract the output
+    images_list = available_images[0].strip().split('\n') if isinstance(available_images, tuple) else available_images.strip().split('\n')
+    images_list = [img for img in images_list if img.strip()]  # Remove empty strings
+
+    # Create the destination directory if it doesn't exist
+    run_cmd(session, "mkdir -p /var/lib/libguestfs/images/")
+
+    # Iterate from the last image backwards
+    for image_name in reversed(images_list):
+        image_name = image_name.strip()
+        if not image_name:
+            continue
+        
+        logger.info(f"Processing image: {image_name}")
+        
+        # wget operation with baseurl + image name
+        image_url = f"{image_base_url.rstrip('/')}/{image_name}"
+        run_cmd(session, f"wget -q {image_url}")
+        
+        # Move the downloaded image to /var/lib/libguestfs/images/
+        run_cmd(session, f"mv {image_name} /var/lib/libguestfs/images/")
+        
+        # chmod 644 on all qcow2 images in the directory
+        run_cmd(session, "chmod 644 /var/lib/libguestfs/images/*.qcow2")
+        
+        # Check kernel version using guestfish
+        image_path = f"/var/lib/libguestfs/images/{image_name}"
+        kernel_version, _ = run_cmd(session, f'virt-ls -a "{image_path}" /usr/lib/modules/')
+        logger.info(f"[INFO] Kernel version for {image_name}: {kernel_version}")
+
+        if version_constraints:
+
+            kernel_ver_str = kernel_version[0].strip() if isinstance(kernel_version, tuple) else kernel_version.strip()
+            
+            # Normalize both versions
+            kernel_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", kernel_ver_str).group(1)
+            constraint_normalized = re.match(r"(\d+\.\d+\.\d+-\d+)", version_constraints).group(1)
+            
+            logger.info(f"[INFO] Comparing kernel {kernel_ver_str} (normalized: {kernel_normalized}) with constraint {version_constraints} (normalized: {constraint_normalized})")
+            
+            # Use LooseVersion for comparison
+            if LooseVersion(kernel_normalized) <= LooseVersion(constraint_normalized):
+                logger.info(f"[INFO] Kernel version meets constraint. Returning image URL: {image_url}")
+                return image_url
+            else:
+                logger.info(f"[INFO] Kernel version does NOT meet constraint, trying next image")
+                run_cmd(session, f"rm -f {image_path}")
+                continue
+        else:
+            logger.info(f"[INFO] No version constraints specified, returning image URL: {image_url}")
+            return image_url

ci_utils/gpfs/gpfs_setup.py

@@ -119,10 +119,26 @@ def _install_on_node(node, sess):
                 run_cmd(sess, "dnf install -y kernel-devel-$(uname -r) kernel-headers-$(uname -r)")
             else:
                 logger.info(f"Attempting Koji fetch...")
-                run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-devel-5.14.0-639.el9.x86_64.rpm")
-                run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/639.el9/x86_64/kernel-headers-5.14.0-639.el9.x86_64.rpm")
-                run_cmd(sess, "ls -la")
-                run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-639.el9.x86_64.rpm ./kernel-headers-5.14.0-639.el9.x86_64.rpm")
+                # Get kernel version and extract components for Koji URL
+                # Example: 5.14.0-658.el9.x86_64 -> major=5.14.0, build=658, release=el9
+                kernel_ver, _ = run_cmd(sess, "uname -r")
+                kernel_ver = kernel_ver.strip()
+                # Parse: 5.14.0-658.el9.x86_64 -> extract 5.14.0, 658, el9
+                match = re.match(r'(\d+\.\d+\.\d+)-(\d+)\.(el\d+)', kernel_ver)
+                if match:
+                    major, build, release = match.groups()
+                    koji_base = f"https://kojihub.stream.centos.org/kojifiles/packages/kernel/{major}/{build}.{release}/x86_64"
+                    kernel_pkg_ver = f"{major}-{build}.{release}"
+                    run_cmd(sess, f"wget {koji_base}/kernel-devel-{kernel_pkg_ver}.x86_64.rpm")
+                    run_cmd(sess, f"wget {koji_base}/kernel-headers-{kernel_pkg_ver}.x86_64.rpm")
+                    run_cmd(sess, "ls -la")
+                    run_cmd(sess, f"dnf -y install openssl-fips-provider ./kernel-devel-{kernel_pkg_ver}.x86_64.rpm ./kernel-headers-{kernel_pkg_ver}.x86_64.rpm")
+                else:
+                    logger.warning(f"Could not parse kernel version {kernel_ver}, falling back to default")
+                    run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-devel-5.14.0-570.el9.x86_64.rpm")
+                    run_cmd(sess, "wget https://kojihub.stream.centos.org/kojifiles/packages/kernel/5.14.0/570.el9/x86_64/kernel-headers-5.14.0-570.el9.x86_64.rpm")
+                    run_cmd(sess, "ls -la")
+                    run_cmd(sess, "dnf -y install openssl-fips-provider ./kernel-devel-5.14.0-570.el9.x86_64.rpm ./kernel-headers-5.14.0-570.el9.x86_64.rpm")
             run_cmd(
                 sess,
                 "yum -y install "

tests/test_pynfs_cthon.py

@@ -367,6 +367,7 @@ def test_pynfs_gpfs(create_session, cmake_flags):
         vm_name = "centos9-vm"
         username = "root"
         ssh_key = "/root/.ssh/id_rsa.pub"
+        version_constraints = "5.14.0-570.62.1.el9_6" #Assuming GPFS 6.0 https://www.ibm.com/docs/en/STXKQY/gpfsclustersfaq.html#fsi
         gerrit_host = os.getenv("GERRIT_HOST", "review.gerrithub.io")
         gerrit_project = os.getenv("GERRIT_PROJECT", "ffilz/nfs-ganesha")
         gerrit_refspec = os.getenv("GERRIT_REFSPEC", "")
@@ -386,13 +387,16 @@ def test_pynfs_gpfs(create_session, cmake_flags):
         # -----------------------
         # VM Setup
         # -----------------------
+        image_url = identify_matching_qcow_image(server_session, "x86_64", "9", "https://cloud.centos.org/centos/9-stream/x86_64/images/", version_constraints=version_constraints)
+        logger.info("Image URL: %s", image_url)
+
         logger.info("Setting up VM on baremetal node: %s", server_node)
         vm = VMManager(
             session=server_session,
             workspace=server_workspace,
             vm_name=vm_name,
-            image_url="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2",
-            image_name="CentOS-Stream-GenericCloud-x86_64-9-20251117.0.x86_64.qcow2",
+            image_url=image_url,
+            image_name=image_url.split("/")[-1],
             vm_cpu="2",
             vm_ram="8192",
             vm_disk="30G",