Inner dev loop fixes

Author: fjakobs

default_python/requirements-dev.txt

@@ -3,6 +3,9 @@
 ## For defining dependencies used by jobs in Databricks Workflows, see
 ## https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
 
+## Add code completion support for DLT
+databricks-dlt
+
 ## pytest is the default package used for testing
 pytest
 

default_python/src/default_python/main.py

@@ -1,11 +1,10 @@
 from pyspark.sql import SparkSession
 
-def get_taxis():
-  spark = SparkSession.builder.getOrCreate()
+def get_taxis(spark: SparkSession):
   return spark.read.table("samples.nyctaxi.trips")
 
 def main():
-  get_taxis().show(5)
+  get_taxis(spark).show(5)
 
 if __name__ == '__main__':
   main()

default_python/src/dlt_pipeline.ipynb

@@ -19,7 +19,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 0,
+   "execution_count": null,
    "metadata": {
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
@@ -33,14 +33,13 @@
    "source": [
     "# Import DLT and src/default_python\n",
     "import dlt\n",
-    "import sys\n",
     "from pyspark.sql.functions import expr\n",
     "from default_python import main"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 0,
+   "execution_count": null,
    "metadata": {
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {},
@@ -54,7 +53,7 @@
    "source": [
     "@dlt.view\n",
     "def taxi_raw():\n",
-    "  return main.get_taxis()\n",
+    "  return main.get_taxis(spark)\n",
     "\n",
     "@dlt.table\n",
     "def filtered_taxis():\n",
@@ -79,7 +78,7 @@
   },
   "language_info": {
    "name": "python",
-   "version": "3.11.4"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

default_python/src/notebook.ipynb

@@ -19,7 +19,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 0,
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {
     "application/vnd.databricks.v1+cell": {
      "cellMetadata": {
@@ -36,7 +46,7 @@
    "source": [
     "from default_python import main\n",
     "\n",
-    "main.get_taxis().show(10)"
+    "main.get_taxis(spark).show(10)"
    ]
   }
  ],
@@ -56,8 +66,16 @@
    "name": "python3"
   },
   "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
    "name": "python",
-   "version": "3.11.4"
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

default_python/tests/main_test.py

@@ -1,14 +1,15 @@
-from databricks.connect import DatabricksSession
-from pyspark.sql import SparkSession
+from databricks.connect import DatabricksSession as SparkSession
 from default_python import main
+from pytest import fixture
 
 # Create a new Databricks Connect session. If this fails,
 # check that you have configured Databricks Connect correctly.
 # See https://docs.databricks.com/dev-tools/databricks-connect.html.
 
-SparkSession.builder = DatabricksSession.builder
-SparkSession.builder.getOrCreate()
+@fixture(scope="session")
+def spark():
+    return SparkSession.builder.getOrCreate()
 
-def test_main():
-    taxis = main.get_taxis()
+def test_main(spark):
+    taxis = main.get_taxis(spark)
     assert taxis.count() > 5