test: revert e2e workflow

Author: kryanbeane

.github/workflows/e2e_tests.yaml

@@ -5,22 +5,21 @@ on:
   pull_request:
     branches:
       - main
-      - "release-*"
+      - 'release-*'
       - ray-jobs-feature
-      - kueue-integration
     paths-ignore:
-      - "docs/**"
-      - "**.adoc"
-      - "**.md"
-      - "LICENSE"
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
 
 concurrency:
   group: ${{ github.head_ref }}-${{ github.workflow }}
   cancel-in-progress: true
 
 env:
-  CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
   KUEUE_VERSION: "v0.13.4"
+  KUBERAY_VERSION: "v1.4.0"
 
 jobs:
   kubernetes:
@@ -35,32 +34,15 @@ jobs:
       - name: Checkout common repo code
         uses: actions/checkout@v4
         with:
-          repository: "project-codeflare/codeflare-common"
-          ref: "main"
-          path: "common"
-
-      - name: Checkout CodeFlare operator repository
-        uses: actions/checkout@v4
-        with:
-          repository: project-codeflare/codeflare-operator
-          path: codeflare-operator
-
-      - name: Set Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: "./codeflare-operator/go.mod"
-          cache-dependency-path: "./codeflare-operator/go.sum"
-
-      - name: Set up gotestfmt
-        uses: gotesttools/gotestfmt-action@v2
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          repository: 'project-codeflare/codeflare-common'
+          ref: 'main'
+          path: 'common'
 
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
-          cache: "pip" # caching pip dependencies
+          python-version: '3.11'
+          cache: 'pip' # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD
         uses: ./common/github-actions/nvidia-gpu-setup
@@ -73,48 +55,12 @@ jobs:
       - name: Install NVidia GPU operator for KinD
         uses: ./common/github-actions/nvidia-gpu-operator
 
-      - name: Wait for nodes to be ready
-        run: |
-          echo "Waiting for all nodes to be ready..."
-          kubectl wait --for=condition=Ready nodes --all --timeout=300s
-
-          echo "Checking node status..."
-          kubectl get nodes -o wide
-
-          echo "Checking for CNI readiness..."
-          for i in {1..30}; do
-            if kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' | grep -v "True"; then
-              echo "Waiting for CNI to initialize (attempt $i/30)..."
-              sleep 10
-            else
-              echo "All nodes are ready!"
-              break
-            fi
-          done
-
-          # Final verification
-          kubectl describe nodes | grep -E "Ready|NetworkReady|RuntimeReady|PodCIDR"
-
-      - name: Deploy CodeFlare stack
+      - name: Deploy KubeRay and Kueue
         id: deploy
         run: |
-          cd codeflare-operator
-          echo Setting up CodeFlare stack
-          make setup-e2e
-          echo Deploying CodeFlare operator
-          make deploy -e IMG="${CODEFLARE_OPERATOR_IMG}" -e ENV="e2e"
-          kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
-          cd ..
-
-      - name: Verify CodeFlare deployment
-        run: |
-          # Wait for Kueue to be ready
-          echo "Waiting for Kueue controller to be ready..."
-          kubectl wait --for=condition=Available --timeout=300s deployment -n kueue-system kueue-controller-manager || {
-            echo "Kueue deployment status:"
-            kubectl get all -n kueue-system
-            exit 1
-          }
+          # Deploy KubeRay operator
+          echo "Deploying KubeRay ${KUBERAY_VERSION}..."
+          kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s"
 
           # Wait for KubeRay to be ready
           echo "Waiting for KubeRay operator to be ready..."
@@ -124,44 +70,43 @@ jobs:
             exit 1
           }
 
-          # Verify webhook certificates
-          echo "Checking CodeFlare operator webhook certificates..."
-          kubectl get secret -n openshift-operators codeflare-operator-webhook-server-cert -o jsonpath='{.data.ca\.crt}' | base64 -d | openssl x509 -noout -text || {
-            echo "Warning: Webhook certificate might be missing or invalid"
+          # Deploy Kueue
+          echo "Deploying Kueue ${KUEUE_VERSION}..."
+          kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/${KUEUE_VERSION}/manifests.yaml
+
+          # Wait for Kueue to be ready
+          echo "Waiting for Kueue controller to be ready..."
+          kubectl wait --for=condition=Available --timeout=300s deployment -n kueue-system kueue-controller-manager || {
+            echo "Kueue deployment status:"
+            kubectl get all -n kueue-system
+            exit 1
           }
 
+          echo "✓ KubeRay and Kueue deployed successfully"
+
       - name: Add user to KinD
         uses: ./common/github-actions/kind-add-user
         with:
           user-name: sdk-user
 
       - name: Configure RBAC for sdk user with limited permissions
         run: |
-          # CRD permissions for discovering resource types
-          kubectl create clusterrole crd-reader --verb=get,list,watch --resource=customresourcedefinitions.apiextensions.k8s.io
-          kubectl create clusterrolebinding sdk-user-crd-reader --clusterrole=crd-reader --user=sdk-user
-
-          # AppWrapper permissions for CodeFlare workloads
-          kubectl create clusterrole appwrapper-creator --verb=get,list,watch,create,update,patch,delete --resource=appwrappers.workload.codeflare.dev
-          kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user
-
-          # Existing permissions
           kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
           kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
           kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
           kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
           kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters.ray.io
           kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
-          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs.ray.io,rayjobs/status
+          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs.ray.io,rayjobs.ray.io/status
           kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
           kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors.kueue.x-k8s.io
           kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
           kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues.kueue.x-k8s.io
           kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
           kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues.kueue.x-k8s.io
           kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
-          kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads.kueue.x-k8s.io
-          kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
+          kubectl create clusterrole workload-reader --verb=get,list,watch --resource=workloads.kueue.x-k8s.io
+          kubectl create clusterrolebinding sdk-user-workload-reader --clusterrole=workload-reader --user=sdk-user
           kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
           kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
           kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
@@ -174,86 +119,40 @@ jobs:
           kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
           kubectl config use-context sdk-user
 
-      - name: Verify cluster readiness before tests
-        run: |
-          echo "=== Pre-test cluster verification ==="
-          echo "Current context:"
-          kubectl config current-context
-
-          echo -e "\nNode status:"
-          kubectl get nodes -o wide
-
-          echo -e "\nSystem pods status:"
-          kubectl get pods -A | grep -E "(kube-system|ray-system|kueue-system|openshift-operators)" || true
-
-          echo -e "\nChecking for any pods in error state:"
-          kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded | grep -v "NAMESPACE" || echo "No pods in error state"
-
-          echo -e "\nKueue resources:"
-          kubectl get resourceflavors,clusterqueues,localqueues -A || true
-
-          echo -e "\nRay CRDs:"
-          kubectl get crd | grep ray || true
-
       - name: Run e2e tests
         run: |
-          export CODEFLARE_TEST_OUTPUT_DIR="${{ runner.temp }}/test-logs"
-          mkdir -p ${CODEFLARE_TEST_OUTPUT_DIR}
+          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
 
           set -euo pipefail
           pip install poetry
           poetry install --with test,docs
           echo "Running e2e tests..."
-          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/e2e-pytest_output.log 2>&1
-        env:
-          GRPC_DNS_RESOLVER: "native"
-
-      - name: Run RayJob e2e tests
-        run: |
-          set -euo pipefail
-          echo "Running RayJob e2e tests..."
-          # Set environment variable to prevent default queue assignment for non-Kueue tests
-          export DISABLE_DEFAULT_KUEUE_QUEUE=true
-
-          # Run only the tests that are designed for Kueue integration
-          poetry run pytest -v -s ./tests/e2e/rayjob/ -x > ${CODEFLARE_TEST_OUTPUT_DIR}/rayjob_pytest_output.log 2>&1
+          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
         env:
           GRPC_DNS_RESOLVER: "native"
 
       - name: Switch to kind-cluster context to print logs
         if: always() && steps.deploy.outcome == 'success'
         run: kubectl config use-context kind-cluster
 
-      - name: Print RayJob E2E Pytest output log
-        if: always() && steps.deploy.outcome == 'success'
-        run: |
-          echo "Printing RayJob Pytest output logs"
-          cat ${CODEFLARE_TEST_OUTPUT_DIR}/rayjob_pytest_output.log || echo "No RayJob test output found"
-
-      - name: Print E2E Pytest output log
-        if: always() && steps.deploy.outcome == 'success'
-        run: |
-          echo "Printing E2E Pytest output logs"
-          cat ${CODEFLARE_TEST_OUTPUT_DIR}/e2e-pytest_output.log || echo "No E2E test output found"
-
-      - name: Print CodeFlare operator logs
+      - name: Print Pytest output log
         if: always() && steps.deploy.outcome == 'success'
         run: |
-          echo "Printing CodeFlare operator logs"
-          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
+          echo "Printing Pytest output logs"
+          cat ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log
 
       - name: Print KubeRay operator logs
         if: always() && steps.deploy.outcome == 'success'
         run: |
           echo "Printing KubeRay operator logs"
-          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+          kubectl logs -n default --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
 
       - name: Export all KinD pod logs
         uses: ./common/github-actions/kind-export-logs
         if: always() && steps.deploy.outcome == 'success'
         with:
-          output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
+          output-directory: ${CODEFLARE_TEST_OUTPUT_DIR}
 
       - name: Upload logs
         uses: actions/upload-artifact@v4
@@ -263,4 +162,3 @@ jobs:
           retention-days: 10
           path: |
             ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
-          if-no-files-found: warn

src/codeflare_sdk/ray/cluster/cluster.py

@@ -209,7 +209,6 @@ def apply(self, force=False):
         namespace = self.config.namespace
         name = self.config.name
 
-        # Regenerate resource_yaml to reflect any configuration changes
         self.resource_yaml = self.create_resource()
 
         try:
@@ -391,23 +390,17 @@ def is_dashboard_ready(self) -> bool:
             bool:
                 True if the dashboard is ready, False otherwise.
         """
-        dashboard_uri = self.cluster_dashboard_uri()
-        if dashboard_uri is None:
-            return False
 
         try:
             response = requests.get(
-                dashboard_uri,
+                self.cluster_dashboard_uri(),
                 headers=self._client_headers,
                 timeout=5,
                 verify=self._client_verify_tls,
             )
         except requests.exceptions.SSLError:  # pragma no cover
             # SSL exception occurs when oauth ingress has been created but cluster is not up
             return False
-        except Exception:  # pragma no cover
-            # Any other exception (connection errors, timeouts, etc.)
-            return False
 
         if response.status_code == 200:
             return True
@@ -536,24 +529,6 @@ def cluster_dashboard_uri(self) -> str:
                         protocol = "https"
                 return f"{protocol}://{ingress.spec.rules[0].host}"
 
-        # For local/test environments without ingress controller (e.g., KIND)
-        # Try to find the Ray head service
-        try:
-            api_instance = client.CoreV1Api(get_api_client())
-            services = api_instance.list_namespaced_service(
-                self.config.namespace,
-                label_selector=f"ray.io/cluster={self.config.name},ray.io/node-type=head",
-            )
-            for service in services.items:
-                if service.metadata.name == f"{self.config.name}-head-svc":
-                    # For ClusterIP services in local environments, return a placeholder
-                    # The actual connection would need port-forwarding or NodePort
-                    return f"http://{service.metadata.name}.{self.config.namespace}.svc.cluster.local:8265"
-        except Exception:  # pragma: no cover
-            pass
-
-        return None
-
     def list_jobs(self) -> List:
         """
         This method accesses the head ray node in your cluster and lists the running jobs.
@@ -813,11 +788,6 @@ def remove_autogenerated_fields(resource):
             else:
                 remove_autogenerated_fields(resource[key])
 
-        # After cleaning, remove empty metadata sections
-        if "metadata" in resource and isinstance(resource["metadata"], dict):
-            if len(resource["metadata"]) == 0:
-                del resource["metadata"]
-
     elif isinstance(resource, list):
         for item in resource:
             remove_autogenerated_fields(item)

src/codeflare_sdk/ray/cluster/test_cluster.py

@@ -322,8 +322,10 @@ def test_cluster_uris(mocker):
     mocker.patch(
         "kubernetes.client.NetworkingV1Api.list_namespaced_ingress",
     )
-    # When no ingress/route/service is found, the method should return None
-    assert cluster.cluster_dashboard_uri() is None
+    assert (
+        cluster.cluster_dashboard_uri()
+        == "Dashboard not available yet, have you run cluster.up()?"
+    )
 
     mocker.patch(
         "codeflare_sdk.ray.cluster.cluster._is_openshift_cluster", return_value=True