RHOAIENG-32532: Fix broken E2E tests

Author: kryanbeane

.github/workflows/e2e_tests.yaml

@@ -19,6 +19,7 @@ concurrency:
 
 env:
   CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
+  KUEUE_VERSION: "v0.13.4"
 
 jobs:
   kubernetes:
@@ -57,7 +58,7 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: 'pip' # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD
@@ -93,8 +94,10 @@ jobs:
           kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
           kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
           kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
-          kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch --resource=rayclusters
+          kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters
           kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
+          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs,rayjobs/status
+          kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
           kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers
           kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
           kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
@@ -111,6 +114,10 @@ jobs:
           kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
           kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
           kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
+          kubectl create clusterrole node-reader --verb=get,list --resource=nodes
+          kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
+          kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads
+          kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
           kubectl config use-context sdk-user
 
       - name: Run e2e tests
@@ -122,19 +129,40 @@ jobs:
           pip install poetry
           poetry install --with test,docs
           echo "Running e2e tests..."
-          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
+          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/e2e-pytest_output.log 2>&1
+        env:
+          GRPC_DNS_RESOLVER: "native"
+
+      - name: Run RayJob e2e tests
+        run: |
+          echo "Running RayJob e2e tests..."
+          # Set environment variable to prevent default queue assignment for non-Kueue tests
+          export DISABLE_DEFAULT_KUEUE_QUEUE=true
+
+          # Install SDK in editable mode if not already done
+          pip install -e .
+
+          # Run only the tests that are designed for Kueue integration
+          poetry run pytest -v -s ./tests/e2e/rayjob/-x >> ${CODEFLARE_TEST_OUTPUT_DIR}/rayjob-e2e-pytest_output.log 2>&1
         env:
           GRPC_DNS_RESOLVER: "native"
 
       - name: Switch to kind-cluster context to print logs
         if: always() && steps.deploy.outcome == 'success'
         run: kubectl config use-context kind-cluster
 
-      - name: Print Pytest output log
+      - name: Print E2E Pytest output log
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing Pytest output logs"
+          cat ${CODEFLARE_TEST_OUTPUT_DIR}/e2e-pytest_output.log
+
+
+      - name: Print RayJob E2E Pytest output log
         if: always() && steps.deploy.outcome == 'success'
         run: |
           echo "Printing Pytest output logs"
-          cat ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log
+          cat ${CODEFLARE_TEST_OUTPUT_DIR}/rayjob-e2e-pytest_output.log
 
       - name: Print CodeFlare operator logs
         if: always() && steps.deploy.outcome == 'success'
@@ -162,3 +190,4 @@ jobs:
           retention-days: 10
           path: |
             ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
+          if-no-files-found: warn

.github/workflows/rayjob_e2e_tests.yaml

@@ -7,14 +7,10 @@
 from ..kubernetes_cluster import config_check, _kube_api_error_handling
 
 
-def get_current_namespace():
+def get_current_namespace():  # pragma: no cover
     """
     Retrieves the current Kubernetes namespace.
 
-    This function attempts to detect the current namespace by:
-    1. First checking if running inside a pod (reading from service account namespace file)
-    2. Falling back to reading from the current kubeconfig context
-
     Returns:
         str:
             The current namespace or None if not found.

src/codeflare_sdk/common/utils/k8s_utils.py

@@ -136,7 +136,6 @@ def build_ray_cluster(cluster: "codeflare_sdk.ray.cluster.Cluster"):
                 "enableIngress": False,
                 "rayStartParams": {
                     "dashboard-host": "0.0.0.0",
-                    "dashboard-port": "8265",
                     "block": "true",
                     "num-gpus": str(head_gpu_count),
                     "resources": head_resources,

src/codeflare_sdk/ray/cluster/build_ray_cluster.py

@@ -783,6 +783,12 @@ def remove_autogenerated_fields(resource):
                 del resource[key]
             else:
                 remove_autogenerated_fields(resource[key])
+
+        # After cleaning, remove empty metadata sections
+        if "metadata" in resource and isinstance(resource["metadata"], dict):
+            if len(resource["metadata"]) == 0:
+                del resource["metadata"]
+
     elif isinstance(resource, list):
         for item in resource:
             remove_autogenerated_fields(item)

src/codeflare_sdk/ray/cluster/cluster.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2025 IBM, Red Hat
+# Copyright 2024 IBM, Red Hat
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

src/codeflare_sdk/ray/cluster/test_config.py

@@ -284,7 +284,6 @@ def _build_head_ray_params(self) -> Dict[str, str]:
         """Build Ray start parameters for head node."""
         params = {
             "dashboard-host": "0.0.0.0",
-            "dashboard-port": "8265",
             "block": "true",
         }