RHOAIENG-32532: Fix broken E2E tests

Author: kryanbeane

.github/workflows/e2e_tests.yaml

@@ -19,6 +19,8 @@ concurrency:
 
 env:
   CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
+  # Explicitly set Docker MTU for KIND to avoid network issues
+  KIND_EXPERIMENTAL_DOCKER_NETWORK: "bridge"
 
 jobs:
   kubernetes:
@@ -60,6 +62,29 @@ jobs:
           python-version: '3.11'
           cache: 'pip' # caching pip dependencies
 
+      - name: Install required tools
+        run: |
+          echo "Installing KIND..."
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-amd64
+          chmod +x ./kind
+          sudo mv ./kind /usr/local/bin/kind
+          kind version
+
+          echo "Installing kubectl if not present..."
+          if ! command -v kubectl &> /dev/null; then
+            curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+            chmod +x kubectl
+            sudo mv kubectl /usr/local/bin/
+          fi
+          kubectl version --client
+
+          echo "Checking for podman..."
+          if ! command -v podman &> /dev/null; then
+            echo "Podman not found, installing..."
+            sudo apt-get update
+            sudo apt-get install -y podman || echo "Podman installation failed, might not be needed"
+          fi
+
       - name: Setup NVidia GPU environment for KinD
         uses: ./common/github-actions/nvidia-gpu-setup
 
@@ -68,9 +93,35 @@ jobs:
         with:
           worker-nodes: 1
 
+      - name: Verify Kind cluster
+        run: |
+          echo "Checking Kind clusters..."
+          kind get clusters
+          echo "Current kubectl context:"
+          kubectl config current-context
+          echo "Checking nodes:"
+          kubectl get nodes
+
+          echo "Waiting for nodes to be ready..."
+          kubectl wait --for=condition=Ready nodes --all --timeout=300s
+
+
       - name: Install NVidia GPU operator for KinD
         uses: ./common/github-actions/nvidia-gpu-operator
 
+      - name: Deploy KubeRay operator
+        run: |
+          KUBERAY_VERSION="v1.2.2"
+          echo "Deploying KubeRay ${KUBERAY_VERSION}"
+          kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
+
+          echo "Waiting for KubeRay operator to become ready..."
+          kubectl wait --for=condition=Available --timeout=300s deployment/kuberay-operator -n ray-system || {
+            echo "KubeRay operator not found, checking pods..."
+            kubectl get pods -A | grep kuberay
+            exit 1
+          }
+
       - name: Deploy CodeFlare stack
         id: deploy
         run: |
@@ -113,11 +164,15 @@ jobs:
           kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
           kubectl config use-context sdk-user
 
-      - name: Run e2e tests
+      - name: Setup test output directory
         run: |
-          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
+          CODEFLARE_TEST_OUTPUT_DIR="${{ runner.temp }}/test-logs"
+          mkdir -p ${CODEFLARE_TEST_OUTPUT_DIR}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
 
+      - name: Run e2e tests
+        run: |
+
           set -euo pipefail
           pip install poetry
           poetry install --with test,docs
@@ -152,7 +207,7 @@ jobs:
         uses: ./common/github-actions/kind-export-logs
         if: always() && steps.deploy.outcome == 'success'
         with:
-          output-directory: ${CODEFLARE_TEST_OUTPUT_DIR}
+          output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
 
       - name: Upload logs
         uses: actions/upload-artifact@v4
@@ -162,3 +217,4 @@ jobs:
           retention-days: 10
           path: |
             ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
+          if-no-files-found: warn

.github/workflows/rayjob_e2e_tests.yaml

@@ -1,4 +1,4 @@
-name: rayjob-e2e-with-kueue
+name: rayjob-e2e
 
 on:
   pull_request:

src/codeflare_sdk/ray/cluster/cluster.py

@@ -783,6 +783,12 @@ def remove_autogenerated_fields(resource):
                 del resource[key]
             else:
                 remove_autogenerated_fields(resource[key])
+
+        # After cleaning, remove empty metadata sections
+        if "metadata" in resource and isinstance(resource["metadata"], dict):
+            if len(resource["metadata"]) == 0:
+                del resource["metadata"]
+
     elif isinstance(resource, list):
         for item in resource:
             remove_autogenerated_fields(item)

tests/e2e/rayjob/ray_version_validation_oauth_test.py

@@ -11,10 +11,7 @@
     ManagedClusterConfig,
 )
 
-# This test validates Ray version compatibility checking for RayJob with cluster lifecycling scenarios
 
-
-@pytest.mark.openshift
 class TestRayJobRayVersionValidationOauth:
     def setup_method(self):
         initialize_kubernetes_client(self)