added support for serverless

Author: mwojtyczka

src/databricks/labs/pytester/fixtures/connect.py

@@ -11,22 +11,29 @@ def spark(ws: WorkspaceClient):
     """
     Get Databricks Connect Spark session. Requires `databricks-connect` package to be installed.
 
+    To enable serverless set the local environment variable DATABRICKS_SERVERLESS_COMPUTE_ID to auto.
+    If this environment variable is set, Databricks Connect ignores the cluster_id.
+
     Usage:
     ```python
     def test_databricks_connect(spark):
         rows = spark.sql("SELECT 1").collect()
         assert rows[0][0] == 1
     ```
     """
-    if not ws.config.cluster_id:
-        skip("No cluster_id found in the environment")
-    ws.clusters.ensure_cluster_is_running(ws.config.cluster_id)
+    if not ws.config.serverless_compute_id:
+        if not ws.config.cluster_id:
+            skip("No cluster_id found in the environment")
+        ws.clusters.ensure_cluster_is_running(ws.config.cluster_id)
+
     try:
         # pylint: disable-next=import-outside-toplevel
         from databricks.connect import (  # type: ignore[import-untyped]
             DatabricksSession,
         )
 
+        if ws.config.serverless_compute_id:
+            return DatabricksSession.builder.sdkConfig(ws.config).serverless(True).getOrCreate()
         return DatabricksSession.builder.sdkConfig(ws.config).getOrCreate()
     except ImportError:
         skip("Please run `pip install databricks-connect`")