Migrate clusters to the new fixture framework (#162)

Author: nfx

tests/integration/conftest.py

@@ -9,7 +9,7 @@
 import pytest
 from databricks.sdk import AccountClient, WorkspaceClient
 from databricks.sdk.core import Config, DatabricksError
-from databricks.sdk.service.compute import ClusterDetails, CreatePolicyResponse
+from databricks.sdk.service.compute import CreatePolicyResponse
 from databricks.sdk.service.iam import AccessControlRequest, PermissionLevel
 from databricks.sdk.service.jobs import CreateResponse
 from databricks.sdk.service.ml import CreateExperimentResponse, ModelDatabricks
@@ -354,41 +354,6 @@ def cluster_policies(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePo
     Threader(executables).run()
 
 
-@pytest.fixture
-def clusters(env: EnvironmentInfo, ws: WorkspaceClient) -> list[ClusterDetails]:
-    logger.debug("Creating test clusters")
-
-    creators = [
-        partial(
-            ws.clusters.create,
-            spark_version=ws.clusters.select_spark_version(latest=True),
-            instance_pool_id=os.environ["TEST_INSTANCE_POOL_ID"],
-            driver_instance_pool_id=os.environ["TEST_INSTANCE_POOL_ID"],
-            cluster_name=f"{env.test_uid}-test-{i}",
-            num_workers=1,
-        )
-        for i in range(NUM_TEST_CLUSTERS)
-    ]
-
-    test_clusters = Threader(creators).run()
-
-    _set_random_permissions(
-        test_clusters,
-        "cluster_id",
-        RequestObjectType.CLUSTERS,
-        env,
-        ws,
-        permission_levels=[PermissionLevel.CAN_ATTACH_TO, PermissionLevel.CAN_MANAGE, PermissionLevel.CAN_RESTART],
-    )
-
-    yield test_clusters
-
-    logger.debug("Deleting test clusters")
-    executables = [partial(ws.clusters.permanent_delete, c.cluster_id) for c in test_clusters]
-    Threader(executables).run()
-    logger.debug("Test clusters deleted")
-
-
 @pytest.fixture
 def experiments(ws: WorkspaceClient, env: EnvironmentInfo) -> list[CreateExperimentResponse]:
     logger.debug("Creating test experiments")
@@ -565,7 +530,6 @@ def workspace_objects(ws: WorkspaceClient, env: EnvironmentInfo) -> WorkspaceObj
 
 @pytest.fixture
 def verifiable_objects(
-    clusters,
     cluster_policies,
     pipelines,
     jobs,
@@ -578,7 +542,6 @@ def verifiable_objects(
     _verifiable_objects = [
         (workspace_objects, "workspace_objects", None),
         (tokens, "tokens", RequestObjectType.AUTHORIZATION),
-        (clusters, "cluster_id", RequestObjectType.CLUSTERS),
         (cluster_policies, "policy_id", RequestObjectType.CLUSTER_POLICIES),
         (pipelines, "pipeline_id", RequestObjectType.PIPELINES),
         (jobs, "job_id", RequestObjectType.JOBS),

tests/integration/test_e2e.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import random
 from typing import Literal
 
@@ -131,6 +132,8 @@ def test_e2e(
     verifiable_objects: list[tuple[list, str, RequestObjectType | None]],
     make_instance_pool,
     make_instance_pool_permissions,
+    make_cluster,
+    make_cluster_permissions,
     make_secret_scope,
     make_secret_scope_acl,
 ):
@@ -145,6 +148,18 @@ def test_e2e(
     )
     verifiable_objects.append(([pool], "instance_pool_id", RequestObjectType.INSTANCE_POOLS))
 
+    cluster = make_cluster(instance_pool_id=os.environ["TEST_INSTANCE_POOL_ID"], single_node=True)
+    make_cluster_permissions(
+        object_id=cluster.cluster_id,
+        permission_level=random.choice(
+            [PermissionLevel.CAN_ATTACH_TO, PermissionLevel.CAN_MANAGE, PermissionLevel.CAN_RESTART]
+        ),
+        group_name=ws_group.display_name,
+    )
+    verifiable_objects.append(
+        ([cluster], "cluster_id", RequestObjectType.CLUSTERS),
+    )
+
     scope = make_secret_scope()
     make_secret_scope_acl(scope=scope, principal=ws_group.display_name, permission=workspace.AclPermission.WRITE)
     verifiable_objects.append(([scope], "secret_scopes", None))