feat: Enable spark-rapids on Dataproc 2.1 Rocky Linux 8

This commit integrates changes to enable the spark-rapids initialization action on Dataproc 2.1-rocky8 images.

- Updates the NVIDIA driver installation process in `spark-rapids.sh` for Rocky Linux:
  - Uses `curl` with retry and fail-fast options for downloading the CUDA installer.
  - Executes the NVIDIA installer with `--silent --driver --toolkit --no-opengl-libs` flags and wraps it in `execute_with_retries`.

- Modifies `test_spark_rapids.py` to enable tests for Rocky Linux on Dataproc 2.1 and below, while keeping them skipped for 2.2+ (Rocky 9).

This resolves the installation issues on Rocky 8. Further work is required to support Rocky 9 (Dataproc 2.2).

Author: cjac

spark-rapids/spark-rapids.sh

@@ -505,11 +505,11 @@ function install_nvidia_gpu_driver() {
     execute_with_retries "dnf install -y kernel-devel-$(uname -r) kernel-headers-$(uname -r)"
 
     # Download the CUDA installer run file
-    curl -o driver.run \
+    curl -fsSL --retry-connrefused --retry 3 --retry-max-time 30 -o driver.run \
         "https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/local_installers/cuda_${CUDA_VERSION}_${NVIDIA_DRIVER_VERSION}_linux.run"
 
     # Run the installer in silent mode
-    bash driver.run --silent
+    execute_with_retries "bash driver.run --silent --driver --toolkit --no-opengl-libs"
 
     # Remove the installer file after installation to clean up
     rm driver.run

spark-rapids/test_spark_rapids.py

@@ -58,8 +58,8 @@ def verify_spark_job_sql(self):
                             ("STANDARD", ["w-0"], GPU_T4))
   def test_spark_rapids(self, configuration, machine_suffixes, accelerator):
 
-    if self.getImageOs() == "rocky":
-      self.skipTest("Not supported for Rocky OS")
+    if self.getImageVersion() > pkg_resources.parse_version("2.0") and self.getImageOs() == "rocky":
+      self.skipTest("Not supported for Rocky 9")
 
     if self.getImageVersion() <= pkg_resources.parse_version("2.0"):
       self.skipTest("Not supported in 2.0 and earlier images")
@@ -88,8 +88,8 @@ def test_spark_rapids(self, configuration, machine_suffixes, accelerator):
                             ("STANDARD", ["w-0"], GPU_T4))
   def test_spark_rapids_sql(self, configuration, machine_suffixes, accelerator):
 
-    if self.getImageOs() == "rocky":
-      self.skipTest("Not supported for Rocky OS")
+    if self.getImageVersion() > pkg_resources.parse_version("2.0") and self.getImageOs() == "rocky":
+      self.skipTest("Not supported for Rocky 9")
 
     if self.getImageVersion() <= pkg_resources.parse_version("2.0"):
       self.skipTest("Not supported in 2.0 and earlier images")
@@ -118,8 +118,8 @@ def test_spark_rapids_sql(self, configuration, machine_suffixes, accelerator):
   def test_non_default_cuda_versions(self, configuration, machine_suffixes,
                                      accelerator, cuda_version, driver_version):
 
-    if self.getImageOs() == "rocky":
-      self.skipTest("Not supported for Rocky OS")
+    if self.getImageVersion() > pkg_resources.parse_version("2.0") and self.getImageOs() == "rocky":
+      self.skipTest("Not supported for Rocky 9")
 
     if self.getImageVersion() <= pkg_resources.parse_version("2.0"):
       self.skipTest("Not supported in 2.0 and earlier images")