Rely UCX on the latest 14.3 LTS DBR instead of 15.x (#1097)

Fixes #1096

This is a quickfix. The long-term solution is tracked in #1098

Author: nfx

src/databricks/labs/ucx/installer/policy.py

@@ -88,8 +88,9 @@ def _get_instance_pool_id(self) -> str | None:
             return None
 
     def _definition(self, conf: dict, instance_profile: str | None, instance_pool_id: str | None) -> str:
+        latest_lts_dbr = self._ws.clusters.select_spark_version(latest=True, long_term_support=True)
         policy_definition = {
-            "spark_version": self._policy_config(self._ws.clusters.select_spark_version(latest=True)),
+            "spark_version": self._policy_config(latest_lts_dbr),
             "node_type_id": self._policy_config(self._ws.clusters.select_node_type(local_disk=True)),
         }
         if instance_pool_id:

tests/unit/installer/test_policy.py

@@ -27,7 +27,7 @@ def common():
     w.cluster_policies.create.return_value = policy
 
     w.cluster_policies.list.return_value = [policy]
-    w.clusters.select_spark_version = lambda latest: "14.2.x-scala2.12"
+    w.clusters.select_spark_version = lambda **_: "14.2.x-scala2.12"
     w.clusters.select_node_type = lambda local_disk: "Standard_F4s"
     w.current_user.me = lambda: iam.User(user_name="[email protected]", groups=[iam.ComplexValue(display="admins")])
     prompts = MockPrompts(

tests/unit/test_install.py

@@ -130,7 +130,7 @@ def download(path: str) -> io.StringIO | io.BytesIO:
     workspace_client.dashboard_widgets.create.return_value = Widget(id="abc")
     workspace_client.clusters.list.return_value = mock_clusters()
     workspace_client.cluster_policies.create.return_value = CreatePolicyResponse(policy_id="foo")
-    workspace_client.clusters.select_spark_version = lambda latest: "14.2.x-scala2.12"
+    workspace_client.clusters.select_spark_version = lambda **_: "14.2.x-scala2.12"
     workspace_client.clusters.select_node_type = lambda local_disk: "Standard_F4s"
     workspace_client.workspace.download = download