added support for serverless

Author: mwojtyczka

README.md

@@ -375,6 +375,9 @@ See also [`log_account_link`](#log_account_link-fixture), [`make_acc_group`](#ma
 ### `spark` fixture
 Get Databricks Connect Spark session. Requires `databricks-connect` package to be installed.
 
+To enable serverless set the local environment variable DATABRICKS_SERVERLESS_COMPUTE_ID to auto.
+If this environment variable is set, Databricks Connect ignores the cluster_id.
+
 Usage:
 ```python
 def test_databricks_connect(spark):

src/databricks/labs/pytester/fixtures/connect.py

@@ -33,7 +33,8 @@ def test_databricks_connect(spark):
         )
 
         if ws.config.serverless_compute_id:
-            return DatabricksSession.builder.sdkConfig(ws.config).serverless(True).getOrCreate()
+            logging.debug("Using serverless compute")
+            return DatabricksSession.builder.serverless(True).getOrCreate()
         return DatabricksSession.builder.sdkConfig(ws.config).getOrCreate()
     except ImportError:
         skip("Please run `pip install databricks-connect`")

tests/integration/conftest.py

@@ -1,3 +1,4 @@
+import os
 import logging
 
 from pytest import fixture
@@ -18,3 +19,10 @@ def debug_env_name():
 @fixture
 def product_info():
     return 'pytester', __version__
+
+
+@fixture
+def serverless_env():
+    os.environ['DATABRICKS_SERVERLESS_COMPUTE_ID'] = 'auto'
+    yield
+    os.environ.pop('DATABRICKS_SERVERLESS_COMPUTE_ID')

tests/integration/fixtures/test_connect.py

@@ -1,3 +1,8 @@
 def test_databricks_connect(spark):
     rows = spark.sql("SELECT 1").collect()
     assert rows[0][0] == 1
+
+
+def test_databricks_connect_serverless(serverless_env, spark):
+    rows = spark.sql("SELECT 1").collect()
+    assert rows[0][0] == 1