Convert pipeline code to use fixtures. (#166)

Author: larsgeorge-db

src/databricks/labs/ucx/providers/mixins/fixtures.py

@@ -250,7 +250,7 @@ def create(
                         permission_level=permission_level,
                     )
                 ]
-            ws.permissions.set(resource_type, object_id, access_control_list=access_control_list)
+            ws.permissions.update(resource_type, object_id, access_control_list=access_control_list)
             return _PermissionsChange(object_id, initial, access_control_list)
 
         def remove(change: _PermissionsChange):
@@ -452,7 +452,7 @@ def create(**kwargs):
 
 @pytest.fixture
 def make_pipeline(ws, make_random, make_notebook):
-    def create(**kwargs):
+    def create(**kwargs) -> pipelines.CreatePipelineResponse:
         if "name" not in kwargs:
             kwargs["name"] = f"sdk-{make_random(4)}"
         if "libraries" not in kwargs:

tests/integration/conftest.py

@@ -14,11 +14,6 @@
 from databricks.sdk.service.jobs import CreateResponse
 from databricks.sdk.service.ml import CreateExperimentResponse, ModelDatabricks
 from databricks.sdk.service.ml import PermissionLevel as ModelPermissionLevel
-from databricks.sdk.service.pipelines import (
-    CreatePipelineResponse,
-    NotebookLibrary,
-    PipelineLibrary,
-)
 from databricks.sdk.service.sql import (
     CreateWarehouseRequestWarehouseType,
     GetWarehouseResponse,
@@ -47,9 +42,7 @@
 
 NUM_TEST_GROUPS = int(os.environ.get("NUM_TEST_GROUPS", 5))
 NUM_TEST_INSTANCE_PROFILES = int(os.environ.get("NUM_TEST_INSTANCE_PROFILES", 3))
-NUM_TEST_CLUSTERS = int(os.environ.get("NUM_TEST_CLUSTERS", 3))
 NUM_TEST_CLUSTER_POLICIES = int(os.environ.get("NUM_TEST_CLUSTER_POLICIES", 3))
-NUM_TEST_PIPELINES = int(os.environ.get("NUM_TEST_PIPELINES", 3))
 NUM_TEST_JOBS = int(os.environ.get("NUM_TEST_JOBS", 3))
 NUM_TEST_EXPERIMENTS = int(os.environ.get("NUM_TEST_EXPERIMENTS", 3))
 NUM_TEST_MODELS = int(os.environ.get("NUM_TEST_MODELS", 3))
@@ -262,36 +255,6 @@ def instance_profiles(env: EnvironmentInfo, ws: WorkspaceClient) -> list[Instanc
     logger.debug("Test instance profiles deleted")
 
 
-@pytest.fixture
-def pipelines(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreatePipelineResponse]:
-    logger.debug("Creating test DLT pipelines")
-
-    test_pipelines: list[CreatePipelineResponse] = [
-        ws.pipelines.create(
-            name=f"{env.test_uid}-test-{i}",
-            continuous=False,
-            development=True,
-            libraries=[PipelineLibrary(notebook=NotebookLibrary(path="/Workspace/sample-notebook"))],
-        )
-        for i in range(NUM_TEST_PIPELINES)
-    ]
-
-    _set_random_permissions(
-        test_pipelines,
-        "pipeline_id",
-        RequestObjectType.PIPELINES,
-        env,
-        ws,
-        permission_levels=[PermissionLevel.CAN_VIEW, PermissionLevel.CAN_RUN, PermissionLevel.CAN_MANAGE],
-    )
-
-    yield test_pipelines
-
-    logger.debug("Deleting test instance pools")
-    executables = [partial(ws.pipelines.delete, p.pipeline_id) for p in test_pipelines]
-    Threader(executables).run()
-
-
 @pytest.fixture
 def jobs(env: EnvironmentInfo, ws: WorkspaceClient) -> list[CreateResponse]:
     logger.debug("Creating test jobs")
@@ -531,7 +494,6 @@ def workspace_objects(ws: WorkspaceClient, env: EnvironmentInfo) -> WorkspaceObj
 @pytest.fixture
 def verifiable_objects(
     cluster_policies,
-    pipelines,
     jobs,
     experiments,
     models,
@@ -543,7 +505,6 @@ def verifiable_objects(
         (workspace_objects, "workspace_objects", None),
         (tokens, "tokens", RequestObjectType.AUTHORIZATION),
         (cluster_policies, "policy_id", RequestObjectType.CLUSTER_POLICIES),
-        (pipelines, "pipeline_id", RequestObjectType.PIPELINES),
         (jobs, "job_id", RequestObjectType.JOBS),
         (experiments, "experiment_id", RequestObjectType.EXPERIMENTS),
         (models, "id", RequestObjectType.REGISTERED_MODELS),

tests/integration/test_e2e.py

@@ -134,6 +134,8 @@ def test_e2e(
     make_instance_pool_permissions,
     make_cluster,
     make_cluster_permissions,
+    make_pipeline,
+    make_pipeline_permissions,
     make_secret_scope,
     make_secret_scope_acl,
 ):
@@ -160,6 +162,16 @@ def test_e2e(
         ([cluster], "cluster_id", RequestObjectType.CLUSTERS),
     )
 
+    pipeline = make_pipeline()
+    make_pipeline_permissions(
+        object_id=pipeline.pipeline_id,
+        permission_level=random.choice([PermissionLevel.CAN_VIEW, PermissionLevel.CAN_RUN, PermissionLevel.CAN_MANAGE]),
+        group_name=ws_group.display_name,
+    )
+    verifiable_objects.append(
+        ([pipeline], "pipeline_id", RequestObjectType.PIPELINES),
+    )
+
     scope = make_secret_scope()
     make_secret_scope_acl(scope=scope, principal=ws_group.display_name, permission=workspace.AclPermission.WRITE)
     verifiable_objects.append(([scope], "secret_scopes", None))