Fix integration test: test_running_real_assessment_job_ext_hms (#3421)

## Changes

This PR updates the `test_running_real_assessment_job_ext_hms`
integration test:

- It ensures that the workflow runs on a cluster with an external HMS,
instead of faking it. (The previous method was insufficient once we
switched to the fast table crawler and the Python4J bridge became
needed.)
- Update the test fixture so that the resources it creates to be crawled
are created in the external HMS, not the main one. Without this the
assessment crawlers partially fail because the resources they are
configured to crawl do not exist on the external HMS.

### Linked issues

Resolves #3134.
Resolves #3246.

### Tests

- updated integration test

Author: asnare

tests/integration/assessment/test_ext_hms.py

@@ -1,30 +1,31 @@
 import dataclasses
 
-from databricks.labs.lsql.backends import CommandExecutionBackend
+import pytest
+
+from databricks.labs.lsql.backends import CommandExecutionBackend, SqlBackend
+from databricks.sdk.service.compute import CreatePolicyResponse
 from databricks.sdk.service.iam import PermissionLevel
 
 
-def test_running_real_assessment_job_ext_hms(
-    ws,
-    installation_ctx,
-    env_or_skip,
-    make_cluster_policy,
-    make_cluster_policy_permissions,
-) -> None:
-    cluster_id = env_or_skip('TEST_EXT_HMS_CLUSTER_ID')
-    ws_group, _ = installation_ctx.make_ucx_group(wait_for_provisioning=True)
-    # TODO: Move `make_cluster_policy` and `make_cluster_policy_permissions` to context like other `make_` methods
-    cluster_policy = make_cluster_policy()
-    make_cluster_policy_permissions(
-        object_id=cluster_policy.policy_id,
-        permission_level=PermissionLevel.CAN_USE,
-        group_name=ws_group.display_name,
-    )
+@pytest.fixture
+def sql_backend(ws, env_or_skip) -> SqlBackend:
+    """Ensure the SQL backend used during fixture setup is attached to the external HMS.
+
+    Various resources are created during setup of the installation context: this ensures that
+    they are created in the external HMS where the assessment workflow will be run. (Otherwise they will not be found.)
+    """
+    cluster_id = env_or_skip("TEST_EXT_HMS_CLUSTER_ID")
+    return CommandExecutionBackend(ws, cluster_id)
+
+
+def test_running_real_assessment_job_ext_hms(installation_ctx, env_or_skip) -> None:
+    main_cluster_id = env_or_skip("TEST_EXT_HMS_NOUC_CLUSTER_ID")
     ext_hms_ctx = installation_ctx.replace(
-        sql_backend=CommandExecutionBackend(ws, cluster_id),
         config_transform=lambda wc: dataclasses.replace(
             wc,
-            override_clusters=None,
+            override_clusters={
+                "main": main_cluster_id,
+            },
             include_object_permissions=[f"cluster-policies:{cluster_policy.policy_id}"],
         ),
         extend_prompts={
@@ -34,14 +35,23 @@ def test_running_real_assessment_job_ext_hms(
             r"Choose a cluster policy": "0",
         },
     )
+    del installation_ctx  # Use ext_hms_ctx from now on instead of installation_ctx to ensure the external HMS is used.
+    ws_group, _ = ext_hms_ctx.make_ucx_group(wait_for_provisioning=True)
+    cluster_policy: CreatePolicyResponse = ext_hms_ctx.make_cluster_policy()
+    ext_hms_ctx.make_cluster_policy_permissions(
+        object_id=cluster_policy.policy_id,
+        permission_level=PermissionLevel.CAN_USE,
+        group_name=ws_group.display_name,
+    )
     ext_hms_ctx.make_linting_resources()
     source_schema = ext_hms_ctx.make_schema(catalog_name="hive_metastore")
     ext_hms_ctx.make_table(schema_name=source_schema.name)
     ext_hms_ctx.workspace_installation.run()
 
     workflow = "assessment"
-    ext_hms_ctx.deployed_workflows.run_workflow(workflow)
-    assert ext_hms_ctx.deployed_workflows.validate_step(workflow), f"Workflow failed: {workflow}"
+    ext_hms_ctx.deployed_workflows.run_workflow(workflow, skip_job_wait=True)
+    workflow_completed_successfully = ext_hms_ctx.deployed_workflows.validate_step(workflow)
+    assert workflow_completed_successfully, f"Workflow failed: {workflow}"
 
     after = ext_hms_ctx.generic_permissions_support.load_as_dict("cluster-policies", cluster_policy.policy_id)
     assert ws_group.display_name in after, f"Group {ws_group.display_name} not found in cluster policy"

tests/integration/hive_metastore/test_ext_hms.py

@@ -13,6 +13,11 @@
 
 @pytest.fixture
 def sql_backend(ws, env_or_skip) -> SqlBackend:
+    """Ensure the SQL backend used during fixture setup is attached to the external HMS.
+
+    Various resources are created during setup of the installation context: this ensures that
+    they are created in the external HMS where the assessment workflow will be run. (Otherwise they will not be found.)
+    """
     cluster_id = env_or_skip("TEST_EXT_HMS_CLUSTER_ID")
     return CommandExecutionBackend(ws, cluster_id)