test: revert e2e workflow

Author: kryanbeane

.github/workflows/e2e_tests.yaml

@@ -5,20 +5,22 @@ on:
   pull_request:
     branches:
       - main
-      - 'release-*'
+      - "release-*"
       - ray-jobs-feature
     paths-ignore:
-      - 'docs/**'
-      - '**.adoc'
-      - '**.md'
-      - 'LICENSE'
+      - "docs/**"
+      - "**.adoc"
+      - "**.md"
+      - "LICENSE"
 
 concurrency:
   group: ${{ github.head_ref }}-${{ github.workflow }}
   cancel-in-progress: true
 
 env:
   CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
+  KUEUE_VERSION: "v0.13.4"
+  KUERAY_VERSION: "v1.4.0"
 
 jobs:
   kubernetes:
@@ -33,9 +35,9 @@ jobs:
       - name: Checkout common repo code
         uses: actions/checkout@v4
         with:
-          repository: 'project-codeflare/codeflare-common'
-          ref: 'main'
-          path: 'common'
+          repository: "project-codeflare/codeflare-common"
+          ref: "main"
+          path: "common"
 
       - name: Checkout CodeFlare operator repository
         uses: actions/checkout@v4
@@ -46,7 +48,7 @@ jobs:
       - name: Set Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: './codeflare-operator/go.mod'
+          go-version-file: "./codeflare-operator/go.mod"
           cache-dependency-path: "./codeflare-operator/go.sum"
 
       - name: Set up gotestfmt
@@ -57,8 +59,8 @@ jobs:
       - name: Set up specific Python version
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
-          cache: 'pip' # caching pip dependencies
+          python-version: "3.11"
+          cache: "pip" # caching pip dependencies
 
       - name: Setup NVidia GPU environment for KinD
         uses: ./common/github-actions/nvidia-gpu-setup
@@ -76,7 +78,7 @@ jobs:
         run: |
           cd codeflare-operator
           echo Setting up CodeFlare stack
-          make setup-e2e
+          make setup-e2e KUEUE_VERSION=${KUEUE_VERSION} KUERAY_VERSION=${KUERAY_VERSION}
           echo Deploying CodeFlare operator
           make deploy -e IMG="${CODEFLARE_OPERATOR_IMG}" -e ENV="e2e"
           kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager

src/codeflare_sdk/ray/cluster/cluster.py

@@ -209,7 +209,6 @@ def apply(self, force=False):
         namespace = self.config.namespace
         name = self.config.name
 
-        # Regenerate resource_yaml to reflect any configuration changes
         self.resource_yaml = self.create_resource()
 
         try:
@@ -391,23 +390,17 @@ def is_dashboard_ready(self) -> bool:
             bool:
                 True if the dashboard is ready, False otherwise.
         """
-        dashboard_uri = self.cluster_dashboard_uri()
-        if dashboard_uri is None:
-            return False
 
         try:
             response = requests.get(
-                dashboard_uri,
+                self.cluster_dashboard_uri(),
                 headers=self._client_headers,
                 timeout=5,
                 verify=self._client_verify_tls,
             )
         except requests.exceptions.SSLError:  # pragma no cover
             # SSL exception occurs when oauth ingress has been created but cluster is not up
             return False
-        except Exception:  # pragma no cover
-            # Any other exception (connection errors, timeouts, etc.)
-            return False
 
         if response.status_code == 200:
             return True
@@ -516,6 +509,8 @@ def cluster_dashboard_uri(self) -> str:
                 ):
                     protocol = "https" if route["spec"].get("tls") else "http"
                     return f"{protocol}://{route['spec']['host']}"
+            # No route found for this cluster
+            return "Dashboard not available yet, have you run cluster.up()?"
         else:
             try:
                 api_instance = client.NetworkingV1Api(get_api_client())
@@ -534,25 +529,9 @@ def cluster_dashboard_uri(self) -> str:
                         protocol = "http"
                     elif "route.openshift.io/termination" in annotations:
                         protocol = "https"
-                return f"{protocol}://{ingress.spec.rules[0].host}"
+                    return f"{protocol}://{ingress.spec.rules[0].host}"
 
-        # For local/test environments without ingress controller (e.g., KIND)
-        # Try to find the Ray head service
-        try:
-            api_instance = client.CoreV1Api(get_api_client())
-            services = api_instance.list_namespaced_service(
-                self.config.namespace,
-                label_selector=f"ray.io/cluster={self.config.name},ray.io/node-type=head",
-            )
-            for service in services.items:
-                if service.metadata.name == f"{self.config.name}-head-svc":
-                    # For ClusterIP services in local environments, return a placeholder
-                    # The actual connection would need port-forwarding or NodePort
-                    return f"http://{service.metadata.name}.{self.config.namespace}.svc.cluster.local:8265"
-        except Exception:  # pragma: no cover
-            pass
-
-        return None
+        return "Dashboard not available yet, have you run cluster.up()?"
 
     def list_jobs(self) -> List:
         """
@@ -813,11 +792,6 @@ def remove_autogenerated_fields(resource):
             else:
                 remove_autogenerated_fields(resource[key])
 
-        # After cleaning, remove empty metadata sections
-        if "metadata" in resource and isinstance(resource["metadata"], dict):
-            if len(resource["metadata"]) == 0:
-                del resource["metadata"]
-
     elif isinstance(resource, list):
         for item in resource:
             remove_autogenerated_fields(item)

src/codeflare_sdk/ray/cluster/test_cluster.py

@@ -322,8 +322,10 @@ def test_cluster_uris(mocker):
     mocker.patch(
         "kubernetes.client.NetworkingV1Api.list_namespaced_ingress",
     )
-    # When no ingress/route/service is found, the method should return None
-    assert cluster.cluster_dashboard_uri() is None
+    assert (
+        cluster.cluster_dashboard_uri()
+        == "Dashboard not available yet, have you run cluster.up()?"
+    )
 
     mocker.patch(
         "codeflare_sdk.ray.cluster.cluster._is_openshift_cluster", return_value=True

tests/e2e/cluster_apply_kind_test.py

@@ -1,8 +1,8 @@
+from calendar import c
+from time import sleep
 from codeflare_sdk import Cluster, ClusterConfiguration
 import pytest
-import time
 from kubernetes import client
-from codeflare_sdk.common.utils import constants
 
 from support import (
     initialize_kubernetes_client,
@@ -40,7 +40,6 @@ def test_cluster_apply(self):
             worker_cpu_limits="1",
             worker_memory_requests="1Gi",
             worker_memory_limits="2Gi",
-            image=f"rayproject/ray:{constants.RAY_VERSION}",
             write_to_file=True,
             verify_tls=False,
         )
@@ -50,9 +49,9 @@ def test_cluster_apply(self):
         cluster.apply()
 
         # Wait for the cluster to be ready
-        cluster.wait_ready(dashboard_check=False)
+        cluster.wait_ready()
         status, ready = cluster.status()
-        assert ready, f"Cluster {cluster_name} is not ready: {status}"
+        assert ready, f"Cluster {cluster_name} is not ready"
 
         # Verify the cluster is created
         ray_cluster = get_ray_cluster(cluster_name, namespace)
@@ -61,7 +60,7 @@ def test_cluster_apply(self):
             ray_cluster["spec"]["workerGroupSpecs"][0]["replicas"] == 1
         ), "Initial worker count does not match"
 
-        # Update configuration with 2 workers
+        # Update configuration with 3 workers
         updated_config = ClusterConfiguration(
             name=cluster_name,
             namespace=namespace,
@@ -74,7 +73,6 @@ def test_cluster_apply(self):
             worker_cpu_limits="1",
             worker_memory_requests="1Gi",
             worker_memory_limits="2Gi",
-            image=f"rayproject/ray:{constants.RAY_VERSION}",
             write_to_file=True,
             verify_tls=False,
         )
@@ -83,15 +81,10 @@ def test_cluster_apply(self):
         cluster.config = updated_config
         cluster.apply()
 
-        # Give Kubernetes a moment to process the update
-        time.sleep(5)
-
         # Wait for the updated cluster to be ready
-        cluster.wait_ready(dashboard_check=False)
+        cluster.wait_ready()
         updated_status, updated_ready = cluster.status()
-        assert (
-            updated_ready
-        ), f"Cluster {cluster_name} is not ready after update: {updated_status}"
+        assert updated_ready, f"Cluster {cluster_name} is not ready after update"
 
         # Verify the cluster is updated
         updated_ray_cluster = get_ray_cluster(cluster_name, namespace)
@@ -101,19 +94,67 @@ def test_cluster_apply(self):
 
         # Clean up
         cluster.down()
+        sleep(10)
+        ray_cluster = get_ray_cluster(cluster_name, namespace)
+        assert ray_cluster is None, "Cluster was not deleted successfully"
 
-        # Wait for deletion to complete (finalizers may delay deletion)
-        max_wait = 30  # seconds
-        wait_interval = 2
-        elapsed = 0
+    def test_apply_invalid_update(self):
+        self.setup_method()
+        create_namespace(self)
 
-        while elapsed < max_wait:
-            ray_cluster = get_ray_cluster(cluster_name, namespace)
-            if ray_cluster is None:
-                break
-            time.sleep(wait_interval)
-            elapsed += wait_interval
+        cluster_name = "test-cluster-apply-invalid"
+        namespace = self.namespace
 
-        assert (
-            ray_cluster is None
-        ), f"Cluster was not deleted successfully after {max_wait}s"
+        # Initial configuration
+        initial_config = ClusterConfiguration(
+            name=cluster_name,
+            namespace=namespace,
+            num_workers=1,
+            head_cpu_requests="500m",
+            head_cpu_limits="1",
+            head_memory_requests="1Gi",
+            head_memory_limits="2Gi",
+            worker_cpu_requests="500m",
+            worker_cpu_limits="1",
+            worker_memory_requests="1Gi",
+            worker_memory_limits="2Gi",
+            write_to_file=True,
+            verify_tls=False,
+        )
+
+        # Create the cluster
+        cluster = Cluster(initial_config)
+        cluster.apply()
+
+        # Wait for the cluster to be ready
+        cluster.wait_ready()
+        status, ready = cluster.status()
+        assert ready, f"Cluster {cluster_name} is not ready"
+
+        # Update with an invalid configuration (e.g., immutable field change)
+        invalid_config = ClusterConfiguration(
+            name=cluster_name,
+            namespace=namespace,
+            num_workers=2,
+            head_cpu_requests="1",
+            head_cpu_limits="2",  # Changing CPU limits (immutable)
+            head_memory_requests="1Gi",
+            head_memory_limits="2Gi",
+            worker_cpu_requests="500m",
+            worker_cpu_limits="1",
+            worker_memory_requests="1Gi",
+            worker_memory_limits="2Gi",
+            write_to_file=True,
+            verify_tls=False,
+        )
+
+        # Try to apply the invalid configuration and expect failure
+        cluster.config = invalid_config
+        cluster.apply()
+
+        cluster.wait_ready()
+        status, ready = cluster.status()
+        assert ready, f"Cluster {cluster_name} is not ready"
+
+        # Clean up
+        cluster.down()

tests/e2e/heterogeneous_clusters_kind_test.py

@@ -48,16 +48,17 @@ def run_heterogeneous_clusters(
                     namespace=self.namespace,
                     num_workers=1,
                     head_cpu_requests="500m",
-                    head_cpu_limits="500m",
-                    head_memory_requests=2,
-                    head_memory_limits=2,
+                    head_cpu_limits="1",
+                    head_memory_requests="1Gi",
+                    head_memory_limits="2Gi",
                     worker_cpu_requests="500m",
                     worker_cpu_limits=1,
-                    worker_memory_requests=1,
-                    worker_memory_limits=4,
+                    worker_memory_requests="1Gi",
+                    worker_memory_limits="2Gi",
                     worker_extended_resource_requests={
                         gpu_resource_name: number_of_gpus
                     },
+                    image=get_ray_image(),
                     write_to_file=True,
                     verify_tls=False,
                     local_queue=queue_name,

tests/e2e/local_interactive_sdk_kind_test.py

@@ -55,17 +55,19 @@ def run_local_interactives(
                 namespace=self.namespace,
                 num_workers=1,
                 head_cpu_requests="500m",
-                head_cpu_limits="500m",
+                head_cpu_limits="1",
+                head_memory_requests="1Gi",
+                head_memory_limits="2Gi",
                 worker_cpu_requests="500m",
                 worker_cpu_limits=1,
-                worker_memory_requests=1,
-                worker_memory_limits=4,
+                worker_memory_requests="1Gi",
+                worker_memory_limits="2Gi",
                 worker_extended_resource_requests={gpu_resource_name: number_of_gpus},
                 verify_tls=False,
             )
         )
 
-        cluster.up()
+        cluster.apply()
 
         cluster.wait_ready()
         cluster.status()

tests/e2e/mnist_raycluster_sdk_aw_kind_test.py

@@ -43,19 +43,21 @@ def run_mnist_raycluster_sdk_kind(
                 namespace=self.namespace,
                 num_workers=1,
                 head_cpu_requests="500m",
-                head_cpu_limits="500m",
+                head_cpu_limits="1",
+                head_memory_requests="1Gi",
+                head_memory_limits="2Gi",
                 worker_cpu_requests="500m",
                 worker_cpu_limits=1,
-                worker_memory_requests=1,
-                worker_memory_limits=4,
+                worker_memory_requests="1Gi",
+                worker_memory_limits="2Gi",
                 worker_extended_resource_requests={gpu_resource_name: number_of_gpus},
                 write_to_file=True,
                 verify_tls=False,
                 appwrapper=True,
             )
         )
 
-        cluster.up()
+        cluster.apply()
 
         cluster.status()