Skip to content

Commit 5b4f896

Browse files
authored
Bundle cli dependencies (#1147)
Bundle terraform binaries with the extension and expose them with env vars to the CLI and the terminal Depends on databricks/cli#1294
1 parent f2fb7a4 commit 5b4f896

File tree

12 files changed

+288
-15
lines changed

12 files changed

+288
-15
lines changed

packages/databricks-vscode/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ bin/**
44
src/test/e2e/workspace/
55
extension/
66
.pytest_cache/
7+
.build/
78

89
# Telemetry file, automatically generated by packages/databricks-vscode/scripts/generateTelemetry.ts
910
telemetry.json

packages/databricks-vscode/.vscodeignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ coverage/
1717
logs/
1818
extension/
1919
**/*.vsix
20+
.build/

packages/databricks-vscode/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,7 @@
776776
"useYarn": false
777777
},
778778
"cli": {
779-
"version": "0.216.0"
779+
"version": "0.217.0"
780780
},
781781
"scripts": {
782782
"vscode:prepublish": "rm -rf out && yarn run package:compile && yarn run package:wrappers:write && yarn run package:jupyter-init-script:write && yarn run package:copy-webview-toolkit && yarn run generate-telemetry",
@@ -787,7 +787,7 @@
787787
"package:darwin:arm64": "./scripts/package-vsix.sh darwin-arm64",
788788
"package:win32:x64": "./scripts/package-vsix.sh win32-x64",
789789
"package:win32:arm64": "./scripts/package-vsix.sh win32-arm64",
790-
"package:all": "yarn run package:linux:x64 && yarn run package:linux:arm64 && yarn run package:darwin:x64 && yarn run package:darwin:arm64 && yarn run package:win32:x64 && yarn run package:win32:arm64",
790+
"package:all": "rm -rf ./.build/ && yarn run package:linux:x64 && yarn run package:linux:arm64 && yarn run package:darwin:x64 && yarn run package:darwin:arm64 && yarn run package:win32:x64 && yarn run package:win32:arm64",
791791
"package:cli:fetch": "bash ./scripts/fetch-databricks-cli.sh ${CLI_ARCH:-}",
792792
"package:cli:link": "rm -f ./bin/databricks && mkdir -p bin && ln -s ../../../../cli/cli bin/databricks",
793793
"package:wrappers:write": "ts-node ./scripts/writeIpynbWrapper.ts -s ./resources/python/notebook.workflow-wrapper.py -o ./resources/python/generated/notebook.workflow-wrapper.json",
Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/bash
22
set -ex
33

44
CLI_VERSION=$(cat package.json | jq -r .cli.version)
@@ -8,6 +8,11 @@ if [ -z "$CLI_ARCH" ]; then
88
CLI_ARCH="$(uname -s | awk '{print tolower($0)}')_$(uname -m)"
99
fi
1010

11+
CLI_DEST=$2
12+
if [ -z "$CLI_DEST" ]; then
13+
CLI_DEST=./bin
14+
fi
15+
1116
CLI_DIR=$(mktemp -d -t databricks-XXXXXXXXXX)
1217
pushd $CLI_DIR
1318
gh release download v${CLI_VERSION} --pattern "databricks_cli_${CLI_VERSION}_${CLI_ARCH}.zip" --repo databricks/cli
@@ -16,8 +21,8 @@ rm databricks_*_$CLI_ARCH.zip
1621
ls
1722

1823
popd
19-
mkdir -p bin
20-
cd ./bin
24+
mkdir -p $CLI_DEST
25+
cd $CLI_DEST
2126
rm -rf databricks
2227
mv $CLI_DIR/databricks* .
2328
rm -rf $CLI_DIR

packages/databricks-vscode/scripts/package-vsix.sh

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,23 @@ case $ARCH in
3838
;;
3939
esac
4040

41+
# Download databricks cli to the .build directory with the correct arch for the environment that runs this script (not the target arch of the vsix)
42+
# This CLI is used to request metadata about CLIs terraform dependencies.
43+
if [ ! -d ./.build ]; then
44+
mkdir ./.build
45+
BUILD_PLATFORM_ARCH="$(uname -s | awk '{print tolower($0)}')_$(uname -m)"
46+
./scripts/fetch-databricks-cli.sh $BUILD_PLATFORM_ARCH ./.build
47+
fi
48+
4149
rm -rf bin
4250
./scripts/fetch-databricks-cli.sh $CLI_ARCH
43-
yarn ts-node ./scripts/set_arch_in_package.ts $VSXI_ARCH -f package.json --cliArch $CLI_ARCH -V $VSXI_ARCH -c $(git rev-parse --short HEAD)
51+
yarn ts-node ./scripts/setArchInPackage.ts $VSXI_ARCH -f package.json --cliArch $CLI_ARCH -V $VSXI_ARCH -c $(git rev-parse --short HEAD)
52+
53+
# Don't bundle terraform for win32-arm64 as they don't support it yet: https://github.com/hashicorp/terraform/issues/32719
54+
if [ $ARCH != "win32-arm64" ]; then
55+
yarn ts-node ./scripts/setupCLIDependencies.ts --cli ./.build/databricks --binDir ./bin --package ./package.json --arch $CLI_ARCH
56+
fi
57+
4458
yarn run prettier package.json --write
4559
TAG="release-v$(cat package.json | jq -r .version)" yarn run package -t $VSXI_ARCH
4660

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import {mkdirp} from "fs-extra";
2+
import assert from "node:assert";
3+
import {spawnSync} from "node:child_process";
4+
import {cp, readFile, writeFile} from "node:fs/promises";
5+
import {tmpdir} from "node:os";
6+
import path from "node:path";
7+
import yargs from "yargs";
8+
import type {
9+
TerraformMetadata,
10+
TerraformMetadataFromCli,
11+
} from "../src/utils/terraformUtils";
12+
13+
async function main() {
14+
const argv = await yargs
15+
.option("cli", {
16+
description: "Path to the Databricks CLI",
17+
type: "string",
18+
requiresArg: true,
19+
})
20+
.option("binDir", {
21+
description: "Path to the bin directory",
22+
type: "string",
23+
requiresArg: true,
24+
})
25+
.option("arch", {
26+
description: "Architecture of databricks cli.",
27+
type: "string",
28+
requiresArg: true,
29+
})
30+
.option("package", {
31+
description: "path/to/package.json",
32+
type: "string",
33+
requiresArg: true,
34+
}).argv;
35+
36+
const res = spawn(argv.cli!, [
37+
"bundle",
38+
"debug",
39+
"terraform",
40+
"--output",
41+
"json",
42+
]);
43+
const dependencies = JSON.parse(res.stdout.toString());
44+
const terraform = dependencies.terraform as TerraformMetadataFromCli;
45+
assert(terraform, "cli must return terraform dependencies");
46+
assert(terraform.version, "cli must return terraform version");
47+
assert(terraform.providerHost, "cli must return provider host");
48+
assert(terraform.providerSource, "cli must return provider source");
49+
assert(terraform.providerVersion, "cli must return provider version");
50+
51+
const tempDir = path.join(tmpdir(), `terraform_${Date.now()}`);
52+
const depsDir = path.join(argv.binDir!, "dependencies");
53+
await mkdirp(tempDir);
54+
await mkdirp(depsDir);
55+
56+
// Download terraform bin for the selected arch
57+
const arch = argv.arch!;
58+
const terraformZip = `terraform_${terraform.version}_${arch}.zip`;
59+
const terraformUrl = `https://releases.hashicorp.com/terraform/${terraform.version}/${terraformZip}`;
60+
spawn("curl", ["-sLO", terraformUrl], {cwd: tempDir});
61+
// Check sha of the archive
62+
const shasumsFile = `terraform_${terraform.version}_SHA256SUMS`;
63+
const shasumsUrl = `https://releases.hashicorp.com/terraform/${terraform.version}/${shasumsFile}`;
64+
spawn("curl", ["-sLO", shasumsUrl], {cwd: tempDir});
65+
const shasumRes = spawn(
66+
"shasum",
67+
["--algorithm", "256", "--check", shasumsFile],
68+
{cwd: tempDir}
69+
);
70+
assert(
71+
shasumRes.output.toString().includes(`${terraformZip}: OK`),
72+
"sha256sum check failed"
73+
);
74+
spawn("unzip", ["-q", terraformZip], {cwd: tempDir});
75+
const fileExt = arch.includes("windows") ? ".exe" : "";
76+
const terraformBinRelPath = path.join(depsDir, `terraform${fileExt}`);
77+
await cp(`${tempDir}/terraform${fileExt}`, terraformBinRelPath);
78+
// Set the path to the terraform bin, the extension will use it to setup the environment variables
79+
const execRelPath = terraformBinRelPath;
80+
81+
// Download databricks provider archive for the selected arch
82+
const providerZip = `terraform-provider-databricks_${terraform.providerVersion}_${arch}.zip`;
83+
spawn(
84+
"gh",
85+
[
86+
"release",
87+
"download",
88+
`v${terraform.providerVersion}`,
89+
"--pattern",
90+
providerZip,
91+
"--repo",
92+
"databricks/terraform-provider-databricks",
93+
],
94+
{cwd: tempDir}
95+
);
96+
const providersMirrorRelPath = path.join(depsDir, "providers");
97+
const databricksProviderDir = path.join(
98+
providersMirrorRelPath,
99+
terraform.providerHost,
100+
terraform.providerSource
101+
);
102+
await mkdirp(databricksProviderDir);
103+
await cp(
104+
path.join(tempDir, providerZip),
105+
path.join(databricksProviderDir, providerZip)
106+
);
107+
// Set the path to the providers mirror dir, the extension will use it
108+
// to create the terraform CLI config at runtime.
109+
const terraformCliConfigRelPath = path.join(depsDir, "config.tfrc");
110+
111+
// Save the info about all dependencies to the package.json
112+
const terraformMetadata: TerraformMetadata = {
113+
...terraform,
114+
execRelPath,
115+
providersMirrorRelPath,
116+
terraformCliConfigRelPath,
117+
};
118+
const rawData = await readFile(argv.package!, {encoding: "utf-8"});
119+
const jsonData = JSON.parse(rawData);
120+
jsonData["terraformMetadata"] = terraformMetadata;
121+
await writeFile(argv.package!, JSON.stringify(jsonData, null, 4), {
122+
encoding: "utf-8",
123+
});
124+
}
125+
126+
function spawn(command: string, args: string[], options: any = {}) {
127+
const child = spawnSync(command, args, options);
128+
if (child.error) {
129+
throw child.error;
130+
} else {
131+
return child;
132+
}
133+
}
134+
135+
main();

packages/databricks-vscode/src/cli/CliWrapper.ts

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,10 @@ export class CliWrapper {
194194
try {
195195
res = await execFile(cmd.command, cmd.args, {
196196
env: {
197-
...EnvVarGenerators.getEnvVarsForCli(configfilePath),
197+
...EnvVarGenerators.getEnvVarsForCli(
198+
this.extensionContext,
199+
configfilePath
200+
),
198201
...EnvVarGenerators.getProxyEnvVars(),
199202
},
200203
});
@@ -295,7 +298,10 @@ export class CliWrapper {
295298
const {stdout, stderr} = await execFile(cmd[0], cmd.slice(1), {
296299
cwd: workspaceFolder.fsPath,
297300
env: {
298-
...EnvVarGenerators.getEnvVarsForCli(configfilePath),
301+
...EnvVarGenerators.getEnvVarsForCli(
302+
this.extensionContext,
303+
configfilePath
304+
),
299305
...EnvVarGenerators.getProxyEnvVars(),
300306
...authProvider.toEnv(),
301307
...this.getLogginEnvVars(),
@@ -340,7 +346,10 @@ export class CliWrapper {
340346
const {stdout, stderr} = await execFile(cmd[0], cmd.slice(1), {
341347
cwd: workspaceFolder.fsPath,
342348
env: {
343-
...EnvVarGenerators.getEnvVarsForCli(configfilePath),
349+
...EnvVarGenerators.getEnvVarsForCli(
350+
this.extensionContext,
351+
configfilePath
352+
),
344353
...EnvVarGenerators.getProxyEnvVars(),
345354
...authProvider.toEnv(),
346355
...this.getLogginEnvVars(),
@@ -369,6 +378,7 @@ export class CliWrapper {
369378
getBundleInitEnvVars(authProvider: AuthProvider) {
370379
return removeUndefinedKeys({
371380
...EnvVarGenerators.getEnvVarsForCli(
381+
this.extensionContext,
372382
workspaceConfigs.databrickscfgLocation
373383
),
374384
...EnvVarGenerators.getProxyEnvVars(),
@@ -425,7 +435,10 @@ export class CliWrapper {
425435

426436
// Add python executable to PATH
427437
const executable = await pythonExtension.getPythonExecutable();
428-
const cliEnvVars = EnvVarGenerators.getEnvVarsForCli(configfilePath);
438+
const cliEnvVars = EnvVarGenerators.getEnvVarsForCli(
439+
this.extensionContext,
440+
configfilePath
441+
);
429442
let shellPath = cliEnvVars.PATH;
430443
if (executable) {
431444
shellPath = `${path.dirname(executable)}${
@@ -491,7 +504,10 @@ export class CliWrapper {
491504
options: SpawnOptionsWithoutStdio;
492505
} {
493506
const env: Record<string, string> = removeUndefinedKeys({
494-
...EnvVarGenerators.getEnvVarsForCli(configfilePath),
507+
...EnvVarGenerators.getEnvVarsForCli(
508+
this.extensionContext,
509+
configfilePath
510+
),
495511
...EnvVarGenerators.getProxyEnvVars(),
496512
...authProvider.toEnv(),
497513
...this.getLogginEnvVars(),

packages/databricks-vscode/src/extension.ts

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@ import {PublicApi} from "@databricks/databricks-vscode-types";
2424
import {LoggerManager, Loggers} from "./logger";
2525
import {logging} from "@databricks/databricks-sdk";
2626
import {workspaceConfigs} from "./vscode-objs/WorkspaceConfigs";
27-
import {FileUtils, PackageJsonUtils, UtilsCommands} from "./utils";
27+
import {
28+
FileUtils,
29+
PackageJsonUtils,
30+
TerraformUtils,
31+
UtilsCommands,
32+
} from "./utils";
2833
import {ConfigureAutocomplete} from "./language/ConfigureAutocomplete";
2934
import {WorkspaceFsCommands, WorkspaceFsDataProvider} from "./workspace-fs";
3035
import {CustomWhenContext} from "./vscode-objs/CustomWhenContext";
@@ -63,6 +68,10 @@ import {TreeItemDecorationProvider} from "./ui/bundle-resource-explorer/Decorati
6368
import {BundleInitWizard} from "./bundle/BundleInitWizard";
6469
import {DatabricksDebugConfigurationProvider} from "./run/DatabricksDebugConfigurationProvider";
6570
import {isIntegrationTest} from "./utils/developmentUtils";
71+
import {getCLIDependenciesEnvVars} from "./utils/envVarGenerators";
72+
73+
// eslint-disable-next-line @typescript-eslint/no-var-requires
74+
const packageJson = require("../package.json");
6675

6776
const customWhenContext = new CustomWhenContext();
6877

@@ -170,6 +179,27 @@ export async function activate(
170179
`${path.delimiter}${context.asAbsolutePath("./bin")}`
171180
);
172181

182+
// We always use bundled terraform and databricks provider.
183+
// Updating environment collection means that the variables will be set in all terminals.
184+
// If users use different CLI version in their terminal it will only pick the variables if
185+
// the dependency versions (that we set together with bin and config paths) match the internal versions of the CLI.
186+
const cliDeps = getCLIDependenciesEnvVars(context);
187+
for (const [key, value] of Object.entries(cliDeps)) {
188+
logging.NamedLogger.getOrCreate(Loggers.Extension).debug(
189+
`Setting env var ${key}=${value}`
190+
);
191+
context.environmentVariableCollection.replace(key, value);
192+
}
193+
TerraformUtils.updateTerraformCliConfig(
194+
context,
195+
packageJson.terraformMetadata
196+
).catch((e) => {
197+
logging.NamedLogger.getOrCreate(Loggers.Extension).error(
198+
"Failed to update terraform cli config",
199+
e
200+
);
201+
});
202+
173203
logging.NamedLogger.getOrCreate(Loggers.Extension).debug("Metadata", {
174204
metadata: packageMetadata,
175205
});

packages/databricks-vscode/src/utils/envVarGenerators.ts

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
import {Loggers} from "../logger";
22
import {readFile} from "fs/promises";
3-
import {Uri} from "vscode";
3+
import {ExtensionContext, Uri} from "vscode";
44
import {logging, Headers} from "@databricks/databricks-sdk";
55
import {ConnectionManager} from "../configuration/ConnectionManager";
66
import {ConfigModel} from "../configuration/models/ConfigModel";
7+
import {TerraformMetadata} from "./terraformUtils";
78

89
// eslint-disable-next-line @typescript-eslint/no-var-requires
9-
const extensionVersion = require("../../package.json").version;
10+
const packageJson = require("../../package.json");
11+
12+
const extensionVersion = packageJson.version;
13+
const terraformMetadata = packageJson.terraformMetadata as TerraformMetadata;
1014

1115
//Get env variables from user's .env file
1216
export async function getUserEnvVars(userEnvPath: Uri) {
@@ -132,7 +136,10 @@ export function getProxyEnvVars() {
132136
};
133137
}
134138

135-
export function getEnvVarsForCli(configfilePath?: string) {
139+
export function getEnvVarsForCli(
140+
extensionContext: ExtensionContext,
141+
configfilePath?: string
142+
) {
136143
/* eslint-disable @typescript-eslint/naming-convention */
137144
return {
138145
HOME: process.env.HOME,
@@ -142,6 +149,25 @@ export function getEnvVarsForCli(configfilePath?: string) {
142149
DATABRICKS_OUTPUT_FORMAT: "json",
143150
DATABRICKS_CLI_UPSTREAM: "databricks-vscode",
144151
DATABRICKS_CLI_UPSTREAM_VERSION: extensionVersion,
152+
...getCLIDependenciesEnvVars(extensionContext),
153+
};
154+
/* eslint-enable @typescript-eslint/naming-convention */
155+
}
156+
157+
export function getCLIDependenciesEnvVars(extensionContext: ExtensionContext) {
158+
if (!terraformMetadata) {
159+
return {};
160+
}
161+
/* eslint-disable @typescript-eslint/naming-convention */
162+
return {
163+
DATABRICKS_TF_VERSION: terraformMetadata.version,
164+
DATABRICKS_TF_EXEC_PATH: extensionContext.asAbsolutePath(
165+
terraformMetadata.execRelPath
166+
),
167+
DATABRICKS_TF_PROVIDER_VERSION: terraformMetadata.providerVersion,
168+
DATABRICKS_TF_CLI_CONFIG_FILE: extensionContext.asAbsolutePath(
169+
terraformMetadata.terraformCliConfigRelPath
170+
),
145171
};
146172
/* eslint-enable @typescript-eslint/naming-convention */
147173
}

0 commit comments

Comments
 (0)