fix unit tests pointing to a databricks workspace outside of vpn

Author: lorenzorubi-db

.github/workflows/flights_liquibase.yml

@@ -103,9 +103,8 @@ jobs:
       - name: Run unit tests
         env:
           DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-          DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
-          DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
-          DBCONNECT_SERVERLESS: "false"
+          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN_TST }}
+          DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
         run: |
           pytest flights/flights-liquibase/tests/ 
       

flights/flights-liquibase/dev-requirements.txt

@@ -1,4 +1,4 @@
 wheel
 pytest
-databricks-sdk[notebook]==0.35.0
-pandas==2.2.3
+databricks-sdk[notebook]>=0.46.0
+pandas==2.2.3

flights/flights-liquibase/liquibase.properties

@@ -1,5 +1,2 @@
-liquibase.command.url: jdbc:databricks://e2-demo-field-eng.cloud.databricks.com:443;AuthMech=3;httpPath=/sql/1.0/warehouses/4b9b953939869799;
-liquibase.databricks.catalog: main
+liquibase.databricks.catalog: liquibase
 liquibase.databricks.schema: lr_liquibase_test
-liquibase.command.username: token
-liquibase.command.password: ${{ secrets.DATABRICKS_TOKEN_TST }}

flights/flights-liquibase/requirements.txt

@@ -1 +1 @@
-databricks-connect==15.4.2
+databricks-connect>=17.0.0

flights/flights-liquibase/tests/unit_transforms/test_flight_transforms.py

@@ -5,17 +5,17 @@
 
 @pytest.fixture(scope="module")
 def spark_session():
-    #try:
-    #    from databricks.connect import DatabricksSession
-    if os.environ.get("DBCONNECT_SERVERLESS", "false").lower() == "true":
-        return DatabricksSession.builder.serverless(True).getOrCreate()
+    try:
+        from databricks.connect import DatabricksSession
+    # if os.environ.get("DBCONNECT_SERVERLESS", "false").lower() == "true":
+    #     return DatabricksSession.builder.serverless(True).getOrCreate()
       #  else:
-      #      return DatabricksSession.builder.getOrCreate()   
+        return DatabricksSession.builder.getOrCreate()
     # except (ValueError, RuntimeError):
     #     from databricks.connect import DatabricksSession
-    #     return DatabricksSession.builder.profile("unit_tests").getOrCreate()    
-    #except (ImportError, ValueError, RuntimeError):
-    else:
+    #     return DatabricksSession.builder.profile("unit_tests").getOrCreate()
+    except (ImportError, ValueError, RuntimeError):
+    # else:
         print("No Databricks Connect, build and return local SparkSession")
         from pyspark.sql import SparkSession
         return SparkSession.builder.getOrCreate()