Bump databricks-sdk-js to v0.5.2 (#898)

kartikgupta-db · web-flow · commit 06d308c7d071 · 2023-10-23T11:35:25.000+02:00
## Changes
&lt;!-- Summary of your changes that are easy to understand --&gt;

## Tests
&lt;!-- How is this tested? --&gt;
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -17,7 +17,7 @@ jobs:
                         { cli_arch: windows_amd64, os: windows-latest },
                     ]
                 node-version: [18.x]
-                vscode-version: [stable, insiders]
+                vscode-version: [stable]
         uses: ./.github/workflows/tests.yml
         with:
             os: ${{ matrix.arch.os }}
diff --git a/package.json b/package.json
@@ -13,7 +13,8 @@
         "yarn": ">=3.2.0"
     },
     "scripts": {
-        "fetch:sdk": "rm -rf vendor && mkdir vendor && gh release download -R databricks/databricks-sdk-js v0.4.0 -p '*.tgz' && mv *.tgz vendor/databricks-sdk.tgz",
+        "install:sdk": "yarn fetch:sdk && chmod a+x scripts/install_sdk.sh && scripts/install_sdk.sh",
+        "fetch:sdk": "rm -rf vendor && mkdir vendor && gh release download -R databricks/databricks-sdk-js v0.5.2 -p '*.tgz' && mv *.tgz vendor/databricks-sdk.tgz",
         "test": "yarn workspaces foreach run test",
         "test:integ": "yarn workspaces foreach run test:integ",
         "build": "yarn workspaces foreach run build",
diff --git a/packages/databricks-vscode-types/package.json b/packages/databricks-vscode-types/package.json
@@ -29,7 +29,7 @@
         "typescript": "5.1.6"
     },
     "dependencies": {
-        "@databricks/databricks-sdk": "../../vendor/databricks-sdk.tgz",
+        "@databricks/databricks-sdk": "file:../../vendor/databricks-sdk.tgz",
         "databricks": "*"
     }
 }
diff --git a/packages/databricks-vscode/package.json b/packages/databricks-vscode/package.json
@@ -681,7 +681,7 @@
         "test:unit": "yarn run build && node ./out/test/runTest.js",
         "test:integ:prepare": "yarn run package",
         "test:integ:run": "wdio run src/test/e2e/wdio.conf.ts",
-        "test:integ:sdk": "ts-mocha --type-check 'src/**/*.integ.ts'",
+        "test:integ:sdk": "ts-mocha --type-check 'src/sdk-extensions/**/*.integ.ts'",
         "test:integ": "yarn run test:integ:prepare && yarn run test:integ:run && yarn run test:integ:sdk",
         "test:cov": "nyc yarn run test:unit",
         "test": "yarn run test:lint && yarn run test:unit",
@@ -690,11 +690,12 @@
         "generate-telemetry": "ts-node ./scripts/generateTelemetry.ts"
     },
     "dependencies": {
-        "@databricks/databricks-sdk": "../../vendor/databricks-sdk.tgz",
+        "@databricks/databricks-sdk": "file:../../vendor/databricks-sdk.tgz",
         "@databricks/databricks-vscode-types": "workspace:^",
         "@vscode/debugadapter": "^1.61.0",
         "@vscode/extension-telemetry": "^0.8.1",
         "@vscode/webview-ui-toolkit": "^1.2.2",
+        "add": "^2.0.6",
         "ansi-to-html": "^0.7.2",
         "bcryptjs": "^2.4.3",
         "triple-beam": "^1.4.1",
diff --git a/packages/databricks-vscode/src/cluster/ClusterManager.test.ts b/packages/databricks-vscode/src/cluster/ClusterManager.test.ts
@@ -8,7 +8,7 @@ import {
     retries,
 } from "@databricks/databricks-sdk";
 import {Cluster} from "../sdk-extensions";
-import {ClusterFixture} from "@databricks/databricks-sdk/dist/test/fixtures";
+import {ClusterFixtures} from "../sdk-extensions/test";
 import {
     anything,
     deepEqual,
@@ -26,7 +26,7 @@ describe(__filename, async () => {
     let testClusterDetails: compute.ClusterDetails;
 
     beforeEach(async () => {
-        ({testClusterDetails} = await ClusterFixture.getMockTestCluster());
+        ({testClusterDetails} = await ClusterFixtures.getMockTestCluster());
         mockedClient = mock(ApiClient);
         when(
             mockedClient.request(
diff --git a/packages/databricks-vscode/src/configuration/ConnectionManager.ts b/packages/databricks-vscode/src/configuration/ConnectionManager.ts
@@ -151,7 +151,7 @@ export class ConnectionManager {
             workspaceClient =
                 projectConfigFile.authProvider.getWorkspaceClient();
 
-            await workspaceClient.config.authenticate({});
+            await workspaceClient.config.authenticate(new Headers());
 
             this._databricksWorkspace = await DatabricksWorkspace.load(
                 workspaceClient,
@@ -285,7 +285,7 @@ export class ConnectionManager {
                 const workspaceClient =
                     config.authProvider.getWorkspaceClient();
 
-                await workspaceClient.config.authenticate({});
+                await workspaceClient.config.authenticate(new Headers());
 
                 await DatabricksWorkspace.load(
                     workspaceClient,
diff --git a/packages/databricks-vscode/src/configuration/auth/MetadataService.test.ts b/packages/databricks-vscode/src/configuration/auth/MetadataService.test.ts
@@ -10,7 +10,6 @@ import {
     MetadataServiceVersionHeader,
     RequestVisitor,
     logging,
-    fetch,
 } from "@databricks/databricks-sdk";
 const {NamedLogger} = logging;
 
@@ -50,7 +49,7 @@ describe(__filename, function () {
                         name: "pat",
                         async configure(): Promise<RequestVisitor> {
                             return async (headers: Headers) => {
-                                headers["Authorization"] = `Bearer XXXX`;
+                                headers.set("Authorization", "Bearer XXXX");
                             };
                         },
                     },
@@ -93,7 +92,7 @@ describe(__filename, function () {
                         name: "pat",
                         async configure(): Promise<RequestVisitor> {
                             return async (headers: Headers) => {
-                                headers["Authorization"] = `Bearer XXXX`;
+                                headers.set("Authorization", "Bearer XXXX");
                             };
                         },
                     },
@@ -109,9 +108,9 @@ describe(__filename, function () {
 
         const apiClient = new ApiClient(config);
 
-        const headers: Record<string, string> = {};
+        const headers = new Headers();
         await apiClient.config.authenticate(headers);
 
-        assert.equal(headers["Authorization"], "Bearer XXXX");
+        assert.equal(headers.get("Authorization"), "Bearer XXXX");
     });
 });
diff --git a/packages/databricks-vscode/src/configuration/auth/MetadataService.ts b/packages/databricks-vscode/src/configuration/auth/MetadataService.ts
@@ -100,12 +100,12 @@ export class MetadataService implements Disposable {
                     res.end(JSON.stringify({bad_version: true}));
                 }
 
-                const headers: Record<string, string> = {};
+                const headers = new Headers();
                 await this._apiClient.config.authenticate(headers);
 
                 res.writeHead(200, {"Content-Type": "text/json"});
 
-                const auth = headers["Authorization"].split(" ");
+                const auth = headers.get("Authorization")!.split(" ");
                 const response: ServerResponse = {
                     access_token: auth[1],
                     expires_on: Math.floor(
diff --git a/packages/databricks-vscode/src/sdk-extensions/Cluster.integ.ts b/packages/databricks-vscode/src/sdk-extensions/Cluster.integ.ts
@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 
-import {CancellationToken, Cluster} from "@databricks/databricks-sdk";
+import {CancellationToken} from "@databricks/databricks-sdk";
+import {Cluster} from "../sdk-extensions";
 import assert from "node:assert";
 import {IntegrationTestSetup} from "./test/IntegrationTestSetup";
 
diff --git a/packages/databricks-vscode/src/sdk-extensions/Cluster.ts b/packages/databricks-vscode/src/sdk-extensions/Cluster.ts
@@ -427,7 +427,9 @@ export class Cluster {
         });
 
         await this.waitForWorkflowCompletion(run, onProgress, token);
-        return await run.getOutput();
+        const output = await run.getOutput();
+        onProgress && onProgress(run.lifeCycleState!, run);
+        return output;
     }
 
     private async waitForWorkflowCompletion(
diff --git a/packages/databricks-vscode/src/sdk-extensions/WorkflowRun.integ.ts b/packages/databricks-vscode/src/sdk-extensions/WorkflowRun.integ.ts
@@ -42,7 +42,7 @@ describe(__filename, function () {
                 },
             });
 
-            assert(progress.length > 1);
+            assert(progress.length >= 1);
             assert.equal(
                 progress[progress.length - 1].lifeCycleState,
                 "TERMINATED"
diff --git a/packages/databricks-vscode/src/sdk-extensions/test/IntegrationTestSetup.ts b/packages/databricks-vscode/src/sdk-extensions/test/IntegrationTestSetup.ts
@@ -38,7 +38,10 @@ export class IntegrationTestSetup {
                 client.apiClient,
                 clusterId
             );
-            await cluster.start();
+            await cluster.start(undefined, (state) =>
+                // eslint-disable-next-line no-console
+                console.log(`Cluster state: ${state}`)
+            );
 
             this._instance = new IntegrationTestSetup(client, cluster);
         }
diff --git a/packages/databricks-vscode/src/sdk-extensions/test/index.ts b/packages/databricks-vscode/src/sdk-extensions/test/index.ts
@@ -0,0 +1,2 @@
+export * as ClusterFixtures from "./ClusterFixtures";
+export * as TokenFixture from "./TokenFixtures";
diff --git a/packages/databricks-vscode/src/test/e2e/run_job_on_cluster_with_repo.e2e.ts b/packages/databricks-vscode/src/test/e2e/run_job_on_cluster_with_repo.e2e.ts
@@ -126,9 +126,9 @@ describe("Run job on cluster with repo", async function () {
                     await browser.getTextByLabel(labelToDefaults.status.label)
                 ).match(/Succeeded/) !== null,
             {
-                timeout: 20000,
-                interval: 50,
-                timeoutMsg: "Job did not reach succeeded status after 20s.",
+                timeout: 30000,
+                interval: 100,
+                timeoutMsg: "Job did not reach succeeded status after 30s.",
             }
         );
 
@@ -201,9 +201,9 @@ describe("Run job on cluster with repo", async function () {
                     await browser.getTextByLabel(labelToDefaults.status.label)
                 ).includes("Succeeded"),
             {
-                timeout: 20000,
+                timeout: 30000,
                 interval: 100,
-                timeoutMsg: "Job did not reach succeeded status after 20s.",
+                timeoutMsg: "Job did not reach succeeded status after 30s.",
             }
         );
 
diff --git a/packages/databricks-vscode/src/test/e2e/run_job_on_cluster_with_workspace.e2e.ts b/packages/databricks-vscode/src/test/e2e/run_job_on_cluster_with_workspace.e2e.ts
@@ -151,9 +151,9 @@ describe("Run job on cluster with workspace", async function () {
                     await browser.getTextByLabel(labelToDefaults.status.label)
                 ).match(/Succeeded/) !== null,
             {
-                timeout: 20000,
-                interval: 50,
-                timeoutMsg: "Job did not reach succeeded status after 20s.",
+                timeout: 30000,
+                interval: 100,
+                timeoutMsg: "Job did not reach succeeded status after 30s.",
             }
         );
 
@@ -225,9 +225,9 @@ describe("Run job on cluster with workspace", async function () {
                     await browser.getTextByLabel(labelToDefaults.status.label)
                 ).match(/Succeeded/) !== null,
             {
-                timeout: 20000,
-                interval: 50,
-                timeoutMsg: "Job did not reach succeeded status after 20s.",
+                timeout: 30000,
+                interval: 100,
+                timeoutMsg: "Job did not reach succeeded status after 30s.",
             }
         );
 
@@ -291,9 +291,9 @@ describe("Run job on cluster with workspace", async function () {
                     await browser.getTextByLabel(labelToDefaults.status.label)
                 ).includes("Succeeded"),
             {
-                timeout: 20000,
+                timeout: 30000,
                 interval: 100,
-                timeoutMsg: "Job did not reach succeeded status after 20s.",
+                timeoutMsg: "Job did not reach succeeded status after 30s.",
             }
         );
 
diff --git a/packages/databricks-vscode/src/test/e2e/wdio.conf.ts b/packages/databricks-vscode/src/test/e2e/wdio.conf.ts
@@ -8,7 +8,12 @@ import path from "node:path";
 import {fileURLToPath} from "url";
 import assert from "assert";
 import fs from "fs/promises";
-import {Config, WorkspaceClient, workspace} from "@databricks/databricks-sdk";
+import {
+    ApiError,
+    Config,
+    WorkspaceClient,
+    workspace,
+} from "@databricks/databricks-sdk";
 import * as ElementCustomCommands from "./customCommands/elementCustomCommands.ts";
 import {execFile, ExecFileOptions} from "node:child_process";
 import {cpSync, mkdirSync, rmSync} from "node:fs";
@@ -614,8 +619,8 @@ async function startCluster(
                 cluster_id: clusterId,
             })
         ).wait();
-    } catch (e: any) {
-        if (e.errorCode !== "INVALID_STATE") {
+    } catch (e: unknown) {
+        if (!(e instanceof ApiError && e.message.includes("INVALID_STATE"))) {
             throw e;
         }
     }
diff --git a/packages/databricks-vscode/src/utils/envVarGenerators.test.ts b/packages/databricks-vscode/src/utils/envVarGenerators.test.ts
@@ -132,9 +132,11 @@ describe(__filename, () => {
             when(mockAuthProvider.authType).thenReturn("profile");
             const mockConfig = mock(Config);
             when(mockApiClient.config).thenReturn(instance(mockConfig));
-            when(mockConfig.authenticate(anything())).thenCall((headers) => {
-                headers["Authorization"] = "Bearer token";
-            });
+            when(mockConfig.authenticate(anything())).thenCall(
+                (headers: Headers) => {
+                    headers.set("Authorization", "Bearer token");
+                }
+            );
             const actual = await getDbConnectEnvVars(
                 instance(mockConnectionManager),
                 mockWorkspacePath
diff --git a/packages/databricks-vscode/src/utils/envVarGenerators.ts b/packages/databricks-vscode/src/utils/envVarGenerators.ts
@@ -75,9 +75,9 @@ export function getCommonDatabricksEnvVars(
 }
 
 async function getPatToken(connectionManager: ConnectionManager) {
-    const headers: Record<string, string> = {};
+    const headers: Headers = new Headers();
     await connectionManager.apiClient?.config.authenticate(headers);
-    return headers["Authorization"]?.split(" ")[1];
+    return headers.get("Authorization")?.split(" ")[1];
 }
 
 async function getSparkRemoteEnvVar(connectionManager: ConnectionManager) {
diff --git a/packages/databricks-vscode/src/workspace-fs/WorkspaceFsWorkflowWrapper.test.ts b/packages/databricks-vscode/src/workspace-fs/WorkspaceFsWorkflowWrapper.test.ts
@@ -146,7 +146,8 @@ describe(__filename, async () => {
                     "RESOURCE_DOES_NOT_EXIST",
                     "RESOURCE_DOES_NOT_EXIST",
                     404,
-                    {}
+                    {},
+                    []
                 )
             );
 
@@ -297,7 +298,8 @@ describe(__filename, async () => {
                     "RESOURCE_DOES_NOT_EXIST",
                     "RESOURCE_DOES_NOT_EXIST",
                     404,
-                    {}
+                    {},
+                    []
                 )
             );
 
diff --git a/scripts/install_sdk.sh b/scripts/install_sdk.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+workspaces="yarn workspaces foreach --exclude '@databricks/databricks-vscode'" #this excludes the root workspace
+
+$workspaces remove @databricks/databricks-sdk;
+
+$workspaces cache clean
+$workspaces cache clean --mirror
+$workspaces add @databricks/databricks-sdk@file:../../vendor/databricks-sdk.tgz
+
+$workspaces cache clean
+$workspaces cache clean --mirror
+$workspaces install
+
+
diff --git a/tsconfig.json b/tsconfig.json
@@ -8,7 +8,7 @@
             "path": "packages/databricks-vscode"
         },
         {
-            "path": "packages/databricks-sdk-js"
+            "path": "packages/databricks-vscode-types"
         }
     ]
 }
diff --git a/vendor/databricks-sdk.tgz b/vendor/databricks-sdk.tgz
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ jobs:`
`17`	`17`	`{ cli_arch: windows_amd64, os: windows-latest },`
`18`	`18`	`]`
`19`	`19`	`node-version: [18.x]`
`20`		`- vscode-version: [stable, insiders]`
	`20`	`+ vscode-version: [stable]`
`21`	`21`	`uses: ./.github/workflows/tests.yml`
`22`	`22`	`with:`
`23`	`23`	`os: ${{ matrix.arch.os }}`
Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@`
`29`	`29`	`"typescript": "5.1.6"`
`30`	`30`	`},`
`31`	`31`	`"dependencies": {`
`32`		`- "@databricks/databricks-sdk": "../../vendor/databricks-sdk.tgz",`
	`32`	`+ "@databricks/databricks-sdk": "file:../../vendor/databricks-sdk.tgz",`
`33`	`33`	`"databricks": "*"`
`34`	`34`	`}`
`35`	`35`	`}`
Original file line number	Diff line number	Diff line change
`@@ -427,7 +427,9 @@ export class Cluster {`
`427`	`427`	`});`
`428`	`428`
`429`	`429`	`await this.waitForWorkflowCompletion(run, onProgress, token);`
`430`		`- return await run.getOutput();`
	`430`	`+ const output = await run.getOutput();`
	`431`	`+ onProgress && onProgress(run.lifeCycleState!, run);`
	`432`	`+ return output;`
`431`	`433`	`}`
`432`	`434`
`433`	`435`	`private async waitForWorkflowCompletion(`