WIP: add serverless e2e tests

Author: ilia-db

packages/databricks-vscode/scripts/list_integration_tests.ts

@@ -21,6 +21,7 @@ const integrationTests = glob
         return {
             path: toUnixPath(path.relative(process.cwd(), testPath)),
             baseName: path.basename(testPath, ".e2e.ts"),
+            serverless: testPath.includes("serverless") ? true : false,
         };
     });
 

packages/databricks-vscode/src/test/e2e/run_files.serverless.e2e.ts

@@ -0,0 +1,56 @@
+import path from "node:path";
+import * as fs from "fs/promises";
+import assert from "node:assert";
+import {
+    dismissNotifications,
+    executeCommandWhenAvailable,
+    openFile,
+    waitForLogin,
+    waitForWorkflowWebview,
+} from "./utils/commonUtils.ts";
+import {sleep} from "wdio-vscode-service";
+import {
+    getBasicBundleConfig,
+    writeRootBundleConfig,
+} from "./utils/dabsFixtures.ts";
+
+describe("Run files on serverless compute", async function () {
+    let projectDir: string;
+    this.timeout(3 * 60 * 1000);
+
+    before(async () => {
+        assert(process.env.WORKSPACE_PATH, "WORKSPACE_PATH doesn't exist");
+
+        projectDir = process.env.WORKSPACE_PATH;
+
+        await fs.writeFile(
+            path.join(projectDir, "lib.py"),
+            [
+                "def func(spark):",
+                `\tspark.sql('SELECT "hello world"').show()`,
+            ].join("\n")
+        );
+        const nestedDir = path.join(projectDir, "nested");
+        await fs.mkdir(nestedDir, {recursive: true});
+        await fs.writeFile(
+            path.join(nestedDir, "hello.py"),
+            [`from lib import func`, "func(spark)"].join("\n")
+        );
+
+        await writeRootBundleConfig(
+            getBasicBundleConfig({}, false),
+            projectDir
+        );
+        await waitForLogin("DEFAULT");
+        await dismissNotifications();
+    });
+
+    beforeEach(async () => {
+        await openFile("hello.py");
+    });
+
+    it("should run a python file as a serverless workflow", async () => {
+        await executeCommandWhenAvailable("Databricks: Run File as Workflow");
+        await waitForWorkflowWebview("hello world");
+    });
+});

packages/databricks-vscode/src/test/e2e/utils/dabsFixtures.ts

@@ -60,10 +60,12 @@ export function getBasicBundleConfig(
     topLevelComputeId = true
 ): BundleSchema {
     assert(process.env.DATABRICKS_HOST, "DATABRICKS_HOST doesn't exist");
-    assert(
-        process.env.TEST_DEFAULT_CLUSTER_ID,
-        "TEST_DEFAULT_CLUSTER_ID doesn't exist"
-    );
+    if (!topLevelComputeId) {
+        assert(
+            process.env.TEST_DEFAULT_CLUSTER_ID,
+            "TEST_DEFAULT_CLUSTER_ID doesn't exist"
+        );
+    }
     /* eslint-disable @typescript-eslint/naming-convention */
     const defaultBundleConfig = {
         bundle: {

packages/databricks-vscode/src/test/e2e/wdio.conf.ts

@@ -315,7 +315,10 @@ export const config: Options.Testrunner = {
             await config.ensureResolved();
 
             assert(config.host, "Config host must be set");
-            assert(config.token, "Config token must be set");
+            assert(
+                config.token || (config.clientId && config.clientSecret),
+                "Config must have a token or a clientId with clientSecret"
+            );
 
             assert(
                 process.env["TEST_DEFAULT_CLUSTER_ID"],
@@ -326,8 +329,13 @@ export const config: Options.Testrunner = {
             console.log(`Creating vscode workspace folder: ${WORKSPACE_PATH}`);
             await fs.mkdir(WORKSPACE_PATH, {recursive: true});
 
-            const client = getWorkspaceClient(config);
-            await startCluster(client, process.env["TEST_DEFAULT_CLUSTER_ID"]);
+            if (config.token) {
+                const client = getWorkspaceClient(config);
+                await startCluster(
+                    client,
+                    process.env["TEST_DEFAULT_CLUSTER_ID"]
+                );
+            }
 
             process.env.DATABRICKS_HOST = config.host!;
             process.env.DATABRICKS_VSCODE_INTEGRATION_TEST = "true";
@@ -599,12 +607,17 @@ export const config: Options.Testrunner = {
 
 async function writeDatabricksConfig(config: Config, rootPath: string) {
     const configFile = path.join(rootPath, ".databrickscfg");
-    await fs.writeFile(
-        configFile,
-        `[DEFAULT]
-host = ${config.host!}
-token = ${config.token!}`
-    );
+    let content = "[DEFAULT]\n";
+    if (config.token) {
+        content += `host = ${config.host!}\n`;
+        content += `token = ${config.token!}\n`;
+    } else {
+        content += `host = ${config.host!}\n`;
+        content += `client_id = ${config.clientId!}\n`;
+        content += `client_secret = ${config.clientSecret!}\n`;
+        content += `serverless_compute_id = auto\n`;
+    }
+    await fs.writeFile(configFile, content);
     return configFile;
 }