Skip to content

Commit 56c6ec0

Browse files
Lokeshchand33michelle0927vunguyenhung
authored
Added Databricks SQL Warehouses API actions (#18148)
* Added Databricks SQL Warehouses API actions - Added create SQL warehouse action - Added list SQL warehouses action - Added get SQL warehouse action - Added edit SQL warehouse action - Added delete SQL warehouse action - Added start/stop SQL warehouse actions - Added get/set SQL warehouse config actions - Added get/set SQL warehouse permissions actions Implements 11 SQL Warehouses endpoints as discussed in the enhancement issue. * Update Databricks SQL Warehouse docs URLs * fix(databricks): bump component versions and apply lint fixes * fix(databricks): addressed requested changes * addressed coderabbit review feedback * resolved the linting issues * addressed all test failures * addressed coderabbit review feedback * resolved the linting issues * addressed coderabbit review feedback * addressed coderabbit review feedback * resolved the linting issues * updates * Add default value for maxNumClusters * create and edit sql warehouses fixes * create and edit sql warehouse fixes * updates --------- Co-authored-by: Michelle Bergeron <[email protected]> Co-authored-by: Leo Vu <[email protected]>
1 parent 9b23afd commit 56c6ec0

File tree

18 files changed

+908
-4
lines changed

18 files changed

+908
-4
lines changed

‎components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import databricks from "../../databricks.app.mjs";
2+
import constants from "../../common/constants.mjs";
3+
import utils from "../../common/utils.mjs";
4+
import { ConfigurationError } from "@pipedream/platform";
5+
6+
export default {
7+
key: "databricks-create-sql-warehouse",
8+
name: "Create SQL Warehouse",
9+
description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)",
10+
version: "0.0.1",
11+
type: "action",
12+
props: {
13+
databricks,
14+
name: {
15+
type: "string",
16+
label: "Warehouse Name",
17+
description: "A human-readable name for the warehouse",
18+
},
19+
clusterSize: {
20+
type: "string",
21+
label: "Cluster Size",
22+
description: "Size of the cluster",
23+
options: constants.CLUSTER_SIZES,
24+
},
25+
autoStopMinutes: {
26+
type: "integer",
27+
label: "Auto Stop (minutes)",
28+
description:
29+
"Minutes of inactivity before auto-stop. 0 disables auto-stop. Must be 0 or ≥ 10.",
30+
optional: true,
31+
default: 10,
32+
},
33+
minNumClusters: {
34+
type: "integer",
35+
label: "Min Number of Clusters",
36+
description: "Minimum number of clusters to maintain (> 0 and ≤ min(max_num_clusters, 30)).",
37+
optional: true,
38+
default: 1,
39+
},
40+
maxNumClusters: {
41+
type: "integer",
42+
label: "Max Number of Clusters",
43+
description: "Maximum number of clusters for autoscaler (≥ min_num_clusters and ≤ 30).",
44+
optional: true,
45+
default: 1,
46+
},
47+
enablePhoton: {
48+
type: "boolean",
49+
label: "Enable Photon",
50+
description: "Whether the warehouse should use Photon optimized clusters.",
51+
optional: true,
52+
},
53+
enableServerlessCompute: {
54+
type: "boolean",
55+
label: "Enable Serverless Compute",
56+
description: "Whether the warehouse should use serverless compute.",
57+
optional: true,
58+
},
59+
warehouseType: {
60+
type: "string",
61+
label: "Warehouse Type",
62+
description:
63+
"Warehouse type: PRO or CLASSIC. Set PRO + enableServerlessCompute = true to use serverless.",
64+
options: [
65+
"TYPE_UNSPECIFIED",
66+
"CLASSIC",
67+
"PRO",
68+
],
69+
optional: true,
70+
},
71+
spotInstancePolicy: {
72+
type: "string",
73+
label: "Spot Instance Policy",
74+
description: "Configures whether the warehouse should use spot instances.",
75+
options: [
76+
"POLICY_UNSPECIFIED",
77+
"COST_OPTIMIZED",
78+
"RELIABILITY_OPTIMIZED",
79+
],
80+
optional: true,
81+
},
82+
channel: {
83+
type: "object",
84+
label: "Channel",
85+
description:
86+
"Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CUSTOM\", \"dbsql_version\": \"2023.35\" }`",
87+
optional: true,
88+
},
89+
tags: {
90+
type: "object",
91+
label: "Tags",
92+
description:
93+
"Custom key-value tags for resources associated with this SQL Warehouse.",
94+
optional: true,
95+
},
96+
instanceProfileArn: {
97+
type: "string",
98+
label: "Instance Profile ARN (Deprecated)",
99+
description: "Deprecated. Instance profile used to pass IAM role to the cluster.",
100+
optional: true,
101+
},
102+
},
103+
104+
async run({ $ }) {
105+
const payload = {
106+
name: this.name,
107+
cluster_size: this.clusterSize,
108+
};
109+
110+
if (this.autoStopMinutes !== undefined) {
111+
if (this.autoStopMinutes !== 0 && this.autoStopMinutes < 10) {
112+
throw new ConfigurationError("autoStopMinutes must be 0 or ≥ 10.");
113+
}
114+
payload.auto_stop_mins = this.autoStopMinutes;
115+
}
116+
117+
const minNumClusters = this.minNumClusters ?? 1;
118+
if (minNumClusters < 1 || minNumClusters > 30) {
119+
throw new ConfigurationError("minNumClusters must be between 1 and 30.");
120+
}
121+
payload.min_num_clusters = minNumClusters;
122+
123+
if (this.maxNumClusters !== undefined) {
124+
if (
125+
this.maxNumClusters < payload.min_num_clusters ||
126+
this.maxNumClusters > 30
127+
) {
128+
throw new ConfigurationError(
129+
`maxNumClusters must be ≥ minNumClusters (${payload.min_num_clusters}) and ≤ 30.`,
130+
);
131+
}
132+
payload.max_num_clusters = this.maxNumClusters;
133+
}
134+
135+
const parsedTags = utils.parseObject(this.tags);
136+
const tagArray = Object.entries(parsedTags).map(([
137+
key,
138+
value,
139+
]) => ({
140+
key,
141+
value,
142+
}));
143+
if (tagArray.length) {
144+
payload.tags = {
145+
custom_tags: tagArray,
146+
};
147+
}
148+
149+
if (this.enablePhoton !== undefined)
150+
payload.enable_photon = this.enablePhoton;
151+
if (this.enableServerlessCompute !== undefined)
152+
payload.enable_serverless_compute = this.enableServerlessCompute;
153+
if (this.warehouseType) payload.warehouse_type = this.warehouseType;
154+
if (this.spotInstancePolicy)
155+
payload.spot_instance_policy = this.spotInstancePolicy;
156+
if (this.channel) payload.channel = utils.parseObject(this.channel);
157+
if (this.instanceProfileArn)
158+
payload.instance_profile_arn = this.instanceProfileArn;
159+
160+
const response = await this.databricks.createSQLWarehouse({
161+
data: payload,
162+
$,
163+
});
164+
165+
$.export(
166+
"$summary",
167+
`Successfully created SQL Warehouse: ${response?.name || this.name}`,
168+
);
169+
return response;
170+
},
171+
};

‎components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import databricks from "../../databricks.app.mjs";
2+
3+
export default {
4+
key: "databricks-delete-sql-warehouse",
5+
name: "Delete SQL Warehouse",
6+
description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)",
7+
version: "0.0.1",
8+
type: "action",
9+
props: {
10+
databricks,
11+
warehouseId: {
12+
description: "The ID of the SQL Warehouse to delete",
13+
propDefinition: [
14+
databricks,
15+
"warehouseId",
16+
],
17+
},
18+
},
19+
async run({ $ }) {
20+
await this.databricks.deleteSQLWarehouse({
21+
warehouseId: this.warehouseId,
22+
$,
23+
});
24+
25+
$.export("$summary", `Successfully deleted SQL Warehouse with ID ${this.warehouseId}`);
26+
return {
27+
success: true,
28+
};
29+
},
30+
};

‎components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
import databricks from "../../databricks.app.mjs";
2+
import constants from "../../common/constants.mjs";
3+
import utils from "../../common/utils.mjs";
4+
import { ConfigurationError } from "@pipedream/platform";
5+
6+
export default {
7+
key: "databricks-edit-sql-warehouse",
8+
name: "Edit SQL Warehouse",
9+
description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)",
10+
version: "0.0.1",
11+
type: "action",
12+
props: {
13+
databricks,
14+
warehouseId: {
15+
description: "The ID of the SQL Warehouse to edit",
16+
propDefinition: [
17+
databricks,
18+
"warehouseId",
19+
],
20+
},
21+
name: {
22+
type: "string",
23+
label: "Warehouse Name",
24+
description: "Logical name for the warehouse. Must be unique within an org and under 100 characters.",
25+
optional: true,
26+
},
27+
clusterSize: {
28+
type: "string",
29+
label: "Cluster Size",
30+
description: "Size of clusters allocated for this warehouse.",
31+
options: constants.CLUSTER_SIZES,
32+
optional: true,
33+
},
34+
autoStopMins: {
35+
type: "integer",
36+
label: "Auto Stop (minutes)",
37+
description: "Minutes of inactivity before auto-stop. 0 disables autostop. Must be 0 or ≥ 10.",
38+
optional: true,
39+
},
40+
minNumClusters: {
41+
type: "integer",
42+
label: "Min Number of Clusters",
43+
description: "Minimum number of available clusters (> 0 and ≤ min(max_num_clusters, 30)).",
44+
optional: true,
45+
},
46+
maxNumClusters: {
47+
type: "integer",
48+
label: "Max Number of Clusters",
49+
description: "Maximum number of clusters for autoscaler (≥ min_num_clusters and ≤ 30).",
50+
optional: true,
51+
},
52+
enablePhoton: {
53+
type: "boolean",
54+
label: "Enable Photon",
55+
description: "Use Photon optimized clusters.",
56+
optional: true,
57+
},
58+
enableServerlessCompute: {
59+
type: "boolean",
60+
label: "Enable Serverless Compute",
61+
description: "Use serverless compute for this warehouse.",
62+
optional: true,
63+
},
64+
warehouseType: {
65+
type: "string",
66+
label: "Warehouse Type",
67+
description: "Set to PRO (recommended) or CLASSIC. Set PRO + enable serverless to use serverless.",
68+
options: [
69+
"TYPE_UNSPECIFIED",
70+
"CLASSIC",
71+
"PRO",
72+
],
73+
optional: true,
74+
},
75+
spotInstancePolicy: {
76+
type: "string",
77+
label: "Spot Instance Policy",
78+
description: "Whether the warehouse should use spot instances.",
79+
options: [
80+
"POLICY_UNSPECIFIED",
81+
"COST_OPTIMIZED",
82+
"RELIABILITY_OPTIMIZED",
83+
],
84+
optional: true,
85+
},
86+
tags: {
87+
type: "object",
88+
label: "Tags",
89+
description: "Key-value tags for all resources associated with this warehouse (fewer than 45 tags).",
90+
optional: true,
91+
},
92+
channel: {
93+
type: "object",
94+
label: "Channel",
95+
description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CUSTOM\", \"dbsql_version\": \"2023.35\" }`",
96+
optional: true,
97+
},
98+
instanceProfileArn: {
99+
type: "string",
100+
label: "Instance Profile ARN (Deprecated)",
101+
description: "Deprecated. Instance profile used to pass IAM role to the cluster.",
102+
optional: true,
103+
},
104+
},
105+
async run({ $ }) {
106+
const payload = {};
107+
108+
if (this.name !== undefined) {
109+
if (typeof this.name !== "string" || this.name.length >= 100) {
110+
throw new ConfigurationError("name must be a string with fewer than 100 characters.");
111+
}
112+
payload.name = this.name;
113+
}
114+
if (this.clusterSize !== undefined) payload.cluster_size = this.clusterSize;
115+
116+
if (this.autoStopMins !== undefined) {
117+
if (this.autoStopMins !== 0 && this.autoStopMins < 10) {
118+
throw new ConfigurationError("autoStopMins must be 0 or >= 10.");
119+
}
120+
payload.auto_stop_mins = this.autoStopMins;
121+
}
122+
123+
if (this.minNumClusters !== undefined) {
124+
if (this.minNumClusters < 1 || this.minNumClusters > 30) {
125+
throw new ConfigurationError("minNumClusters must be between 1 and 30.");
126+
}
127+
payload.min_num_clusters = this.minNumClusters;
128+
}
129+
130+
if (this.maxNumClusters !== undefined) {
131+
if (this.maxNumClusters < 1 || this.maxNumClusters > 30) {
132+
throw new ConfigurationError("maxNumClusters must be between 1 and 30.");
133+
}
134+
if (this.minNumClusters !== undefined && this.maxNumClusters < this.minNumClusters) {
135+
throw new ConfigurationError("maxNumClusters must be >= minNumClusters.");
136+
}
137+
payload.max_num_clusters = this.maxNumClusters;
138+
}
139+
140+
if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton;
141+
if (this.enableServerlessCompute !== undefined) {
142+
if (this.warehouseType === "CLASSIC" && this.enableServerlessCompute) {
143+
throw new ConfigurationError("Serverless compute requires warehouseType = PRO.");
144+
}
145+
payload.enable_serverless_compute = this.enableServerlessCompute;
146+
}
147+
148+
const parsedTags = utils.parseObject(this.tags);
149+
const tagArray = Object.entries(parsedTags).map(([
150+
key,
151+
value,
152+
]) => ({
153+
key,
154+
value,
155+
}));
156+
if (tagArray.length) {
157+
payload.tags = {
158+
custom_tags: tagArray,
159+
};
160+
}
161+
if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType;
162+
if (this.spotInstancePolicy !== undefined) {
163+
payload.spot_instance_policy = this.spotInstancePolicy;
164+
}
165+
if (this.channel !== undefined) payload.channel = utils.parseObject(this.channel);
166+
if (this.instanceProfileArn !== undefined) {
167+
payload.instance_profile_arn = this.instanceProfileArn;
168+
}
169+
170+
if (!Object.keys(payload).length) {
171+
throw new ConfigurationError("No fields to update. Provide at least one property.");
172+
}
173+
174+
const response = await this.databricks.editSQLWarehouse({
175+
warehouseId: this.warehouseId,
176+
data: payload,
177+
$,
178+
});
179+
180+
$.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`);
181+
return response;
182+
},
183+
};

‎components/databricks/actions/get-run-output/get-run-output.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ export default {
44
key: "databricks-get-run-output",
55
name: "Get Run Output",
66
description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)",
7-
version: "0.0.1",
7+
version: "0.0.2",
88
type: "action",
99
props: {
1010
databricks,

0 commit comments

Comments
 (0)