Fix %run magic with databricks notebooks on windows (#1628)

## Changes

On windows `tempfile.NamedTemporaryFile` with `delete=True` (default)
deletes the temp file when it's closed. Internally the `%run` logic
opens and closes the file before execution (to check that it can open
it), which leads to deletion of the file and an error seen in #1624

Here we use lower-level temp file logic and unlink it ourselves after
the execution is fully done.

One more change is an installation of `nbformat` dependency at runtime
(together with `databricks-connect`), which is required for the `%run`
to work.

## Tests

e2e

Author: ilia-db

packages/databricks-vscode/resources/python/00-databricks-init.py

@@ -221,10 +221,14 @@ def databricks_notebook_exec_env(project_root: str, py_file: str):
     try:
         if is_databricks_notebook(py_file):
             notebook = convert_databricks_notebook_to_ipynb(py_file)
-            with tempfile.NamedTemporaryFile(suffix=".ipynb") as f:
-                f.write(notebook.encode())
-                f.flush()
-                yield f.name
+            fd, temp_path = tempfile.mkstemp(suffix=".ipynb")
+            with os.fdopen(fd, 'wb') as temp_file:
+                temp_file.write(notebook.encode())
+            try:
+                yield temp_path
+            finally:
+                if os.path.exists(temp_path):
+                    os.unlink(temp_path)
         else:
             yield py_file
     finally:

packages/databricks-vscode/src/language/EnvironmentDependenciesInstaller.ts

@@ -50,6 +50,12 @@ export class EnvironmentDependenciesInstaller implements Disposable {
                 version,
                 this.outputChannel
             );
+            // Required for executing notebooks with %run magic
+            await this.pythonExtension.installPackageInEnvironment(
+                "nbformat",
+                undefined,
+                this.outputChannel
+            );
         } catch (e: unknown) {
             if (e instanceof Error) {
                 window.showErrorMessage(e.message);

packages/databricks-vscode/src/test/e2e/run_dbconnect.ucws.e2e.ts

@@ -100,6 +100,20 @@ describe("Run files on serverless compute", async function () {
             })
         );
 
+        await fs.writeFile(
+            path.join(nestedDir, "databricks-run-notebook.py"),
+            [
+                "# Databricks notebook source",
+                "# DBTITLE 1,My cell title",
+                "# MAGIC %sql",
+                "# MAGIC select 1 + 1;",
+                "# MAGIC select 'hello run;'",
+                "# COMMAND ----------",
+                `df = _sqldf.toPandas()`,
+                `df.to_json(os.path.join(os.getcwd(), "databricks-run-notebook-output.json"))`,
+            ].join("\n")
+        );
+
         await fs.writeFile(
             path.join(nestedDir, "databricks-notebook.py"),
             [
@@ -114,7 +128,7 @@ describe("Run files on serverless compute", async function () {
                 `df = _sqldf.toPandas()`,
                 `df.to_json(os.path.join(os.getcwd(), "databricks-notebook-output.json"))`,
                 "# COMMAND ----------",
-                "# MAGIC %run './hello.py'",
+                "# MAGIC %run './databricks-run-notebook.py'",
             ].join("\n")
         );
 
@@ -288,8 +302,8 @@ describe("Run files on serverless compute", async function () {
         const runOutputFile = path.join(
             projectDir,
             "nested",
-            "file-output.json"
+            "databricks-run-notebook-output.json"
         );
-        await checkOutputFile(runOutputFile, "hello world");
+        await checkOutputFile(runOutputFile, "hello run;");
     });
 });