From 97b92e1980c7b825b2643fe6a0210fd2677e75a3 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Fri, 22 Aug 2025 17:59:18 +0530 Subject: [PATCH 01/39] Added Databricks SQL Warehouses API actions - Added create SQL warehouse action - Added list SQL warehouses action - Added get SQL warehouse action - Added edit SQL warehouse action - Added delete SQL warehouse action - Added start/stop SQL warehouse actions - Added get/set SQL warehouse config actions - Added get/set SQL warehouse permissions actions Implements 11 SQL Warehouses endpoints as discussed in the enhancement issue. --- .../create-sql-warehouse.mjs | 44 ++++++++++ .../delete-sql-warehouse.mjs | 26 ++++++ .../edit-sql-warehouse/edit-sql-warehouse.mjs | 32 ++++++++ .../get-sql-warehouse-config.mjs | 17 ++++ .../get-sql-warehouse-permissions.mjs | 26 ++++++ .../get-sql-warehouse/get-sql-warehouse.mjs | 26 ++++++ .../list-sql-warehouses.mjs | 38 +++++++++ .../set-sql-warehouse-config.mjs | 26 ++++++ .../set-sql-warehouse-permissions.mjs | 32 ++++++++ .../start-sql-warehouse.mjs | 26 ++++++ .../stop-sql-warehouse/stop-sql-warehouse.mjs | 26 ++++++ components/databricks/databricks.app.mjs | 80 +++++++++++++++++++ 12 files changed, 399 insertions(+) create mode 100644 components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs create mode 100644 components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs create mode 100644 components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs create mode 100644 components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs create mode 100644 components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs create mode 100644 components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs create mode 100644 components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs create mode 100644 components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs create mode 100644 components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs create mode 100644 components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs create mode 100644 components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs new file mode 100644 index 0000000000000..bd54c5b887002 --- /dev/null +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -0,0 +1,44 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-create-sql-warehouse", + name: "Create SQL Warehouse", + description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/create)", + version: "0.0.1", + type: "action", + props: { + databricks, + name: { + type: "string", + label: "Warehouse Name", + description: "A human-readable name for the warehouse", + }, + clusterSize: { + type: "string", + label: "Cluster Size", + description: "Size of the cluster (e.g., `Small`, `Medium`, `Large`)", + }, + autoStopMinutes: { + type: "integer", + label: "Auto Stop (minutes)", + description: "Number of minutes of inactivity before the warehouse auto-stops", + optional: true, + default: 10, + }, + }, + async run({ $ }) { + const payload = { + name: this.name, + cluster_size: this.clusterSize, + auto_stop_mins: this.autoStopMinutes, + }; + + const response = await this.databricks.createSQLWarehouse({ + data: payload, + $, + }); + + $.export("$summary", `Successfully created SQL Warehouse: ${response?.name || this.name}`); + return response; + }, +}; diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs new file mode 100644 index 0000000000000..91c5ff98a7865 --- /dev/null +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-sql-warehouse", + name: "Delete SQL Warehouse", + description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/delete)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to delete", + }, + }, + async run({ $ }) { + await this.databricks.deleteSQLWarehouse({ + warehouseId: this.warehouseId, + $, + }); + + $.export("$summary", `Successfully deleted SQL Warehouse with ID ${this.warehouseId}`); + return { success: true }; + }, +}; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs new file mode 100644 index 0000000000000..ad63557b96e51 --- /dev/null +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -0,0 +1,32 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-edit-sql-warehouse", + name: "Edit SQL Warehouse", + description: "Edits the configuration of an existing SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/edit)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to edit", + }, + body: { + type: "object", + label: "Edit Configuration", + description: "The new configuration for the SQL Warehouse (JSON object). Example: `{ \"name\": \"Updated Warehouse\", \"cluster_size\": \"2X-Small\" }`", + }, + }, + async run({ $ }) { + const response = await this.databricks.editSQLWarehouse({ + warehouseId: this.warehouseId, + data: this.body, + $, + }); + + $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs new file mode 100644 index 0000000000000..c57ceedab62d9 --- /dev/null +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -0,0 +1,17 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-sql-warehouse-config", + name: "Get SQL Warehouse Config", + description: "Retrieves the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouse/get-config)", + version: "0.0.1", + type: "action", + props: { + databricks, + }, + async run({ $ }) { + const response = await this.databricks.getSQLWarehouseConfig({ $ }); + $.export("$summary", "Successfully retrieved SQL Warehouse configuration"); + return response; + }, +}; diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs new file mode 100644 index 0000000000000..331b3157ef15c --- /dev/null +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-sql-warehouse-permissions", + name: "Get SQL Warehouse Permissions", + description: "Retrieves the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/get-permissions)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to fetch permissions for", + }, + }, + async run({ $ }) { + const response = await this.databricks.getSQLWarehousePermissions({ + warehouseId: this.warehouseId, + $, + }); + + $.export("$summary", `Retrieved permissions for SQL Warehouse ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs new file mode 100644 index 0000000000000..4fed49f0f8ac5 --- /dev/null +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-sql-warehouse", + name: "Get SQL Warehouse", + description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/get)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to retrieve", + }, + }, + async run({ $ }) { + const response = await this.databricks.getSQLWarehouse({ + warehouseId: this.warehouseId, + $, + }); + + $.export("$summary", `Retrieved details for SQL Warehouse ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs new file mode 100644 index 0000000000000..28205283bd3e2 --- /dev/null +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -0,0 +1,38 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-list-sql-warehouses", + name: "List SQL Warehouses", + description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/list)", + version: "0.0.1", + type: "action", + props: { + databricks, + maxResults: { + type: "integer", + label: "Max Results", + description: "Maximum number of warehouses to return", + default: 50, + optional: true, + }, + }, + async run({ $ }) { + const params = { + limit: this.maxResults || 50, + }; + + const { warehouses } = await this.databricks.listSQLWarehouses({ + params, + $, + }); + + if (!warehouses?.length) { + $.export("$summary", "No warehouses found."); + return []; + } + + $.export("$summary", `Successfully retrieved ${warehouses.length} warehouse${warehouses.length === 1 ? "" : "s"}.`); + + return warehouses; + }, +}; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs new file mode 100644 index 0000000000000..d7c38fd1094e8 --- /dev/null +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-set-sql-warehouse-config", + name: "Set SQL Warehouse Config", + description: "Updates the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouse/set-config)", + version: "0.0.1", + type: "action", + props: { + databricks, + config: { + type: "object", + label: "Configuration", + description: "The configuration object for SQL Warehouses. Example: `{ \"enable_serverless_compute\": true }`", + }, + }, + async run({ $ }) { + const response = await this.databricks.setSQLWarehouseConfig({ + data: this.config, + $, + }); + + $.export("$summary", "Successfully updated SQL Warehouse configuration"); + return response; + }, +}; diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs new file mode 100644 index 0000000000000..2f1d664c96b58 --- /dev/null +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -0,0 +1,32 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-set-sql-warehouse-permissions", + name: "Set SQL Warehouse Permissions", + description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/set-permissions)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to update permissions for", + }, + permissions: { + type: "object", + label: "Permissions", + description: "The permissions object. Example: `{ \"access_control_list\": [{ \"user_name\": \"alice@example.com\", \"permission_level\": \"CAN_MANAGE\" }] }`", + }, + }, + async run({ $ }) { + const response = await this.databricks.setSQLWarehousePermissions({ + warehouseId: this.warehouseId, + data: this.permissions, + $, + }); + + $.export("$summary", `Successfully updated permissions for SQL Warehouse ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs new file mode 100644 index 0000000000000..1765702cc5fee --- /dev/null +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-start-sql-warehouse", + name: "Start SQL Warehouse", + description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/start)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to start", + }, + }, + async run({ $ }) { + const response = await this.databricks.startSQLWarehouse({ + warehouseId: this.warehouseId, + $, + }); + + $.export("$summary", `Successfully started SQL Warehouse with ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs new file mode 100644 index 0000000000000..58f227c2e8fdd --- /dev/null +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -0,0 +1,26 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-stop-sql-warehouse", + name: "Stop SQL Warehouse", + description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/stop)", + version: "0.0.1", + type: "action", + props: { + databricks, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to stop", + }, + }, + async run({ $ }) { + const response = await this.databricks.stopSQLWarehouse({ + warehouseId: this.warehouseId, + $, + }); + + $.export("$summary", `Successfully stopped SQL Warehouse with ID ${this.warehouseId}`); + return response; + }, +}; diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 6daaf8eec3e84..76420a0d5bdc6 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -90,5 +90,85 @@ export default { ...args, }); }, + createSQLWarehouse(args = {}) { + return this._makeRequest({ + path: "/sql/warehouses", + method: "POST", + ...args, + }); + }, + deleteSQLWarehouse({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/sql/warehouses/${warehouseId}`, + method: "DELETE", + ...args, + }); + }, + getSQLWarehouse({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/sql/warehouses/${warehouseId}`, + method: "GET", + ...args, + }); + }, + listSQLWarehouses(args = {}) { + return this._makeRequest({ + path: "/sql/warehouses", + ...args, + }); + }, + editSQLWarehouse({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/sql/warehouses/${warehouseId}/edit`, + method: "POST", + ...args, + }); + }, + startSQLWarehouse({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/sql/warehouses/${warehouseId}/start`, + method: "POST", + ...args, + }); + }, + + stopSQLWarehouse({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/sql/warehouses/${warehouseId}/stop`, + method: "POST", + ...args, + }); + }, + + getSQLWarehouseConfig(args = {}) { + return this._makeRequest({ + path: "/sql/config/warehouses", + method: "GET", + ...args, + }); + }, + + setSQLWarehouseConfig(args = {}) { + return this._makeRequest({ + path: "/sql/config/warehouses", + method: "PUT", + ...args, + }); + }, + getSQLWarehousePermissions({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/permissions/warehouses/${warehouseId}`, + method: "GET", + ...args, + }); + }, + + setSQLWarehousePermissions({ warehouseId, ...args }) { + return this._makeRequest({ + path: `/permissions/warehouses/${warehouseId}`, + method: "PUT", + ...args, + }); + } }, }; From 5a697bdc0756f96e845394ce539faf6a3c138fd5 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Fri, 22 Aug 2025 22:07:05 +0530 Subject: [PATCH 02/39] Update Databricks SQL Warehouse docs URLs --- .../create-sql-warehouse.mjs | 2 +- .../delete-sql-warehouse.mjs | 2 +- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 2 +- .../get-sql-warehouse-config.mjs | 2 +- .../get-sql-warehouse-permissions.mjs | 2 +- .../get-sql-warehouse/get-sql-warehouse.mjs | 2 +- .../list-sql-warehouses.mjs | 2 +- .../set-sql-warehouse-config.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 23 ++++++++++++++----- .../start-sql-warehouse.mjs | 2 +- .../stop-sql-warehouse/stop-sql-warehouse.mjs | 2 +- 11 files changed, 27 insertions(+), 16 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index bd54c5b887002..3cb6c11f6ad51 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", - description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/create)", + description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 91c5ff98a7865..87278c82a46ee 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-delete-sql-warehouse", name: "Delete SQL Warehouse", - description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/delete)", + description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index ad63557b96e51..4cf5accf2493b 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", - description: "Edits the configuration of an existing SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/edit)", + description: "Edits the configuration of an existing SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/edit)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index c57ceedab62d9..9b060a41ec54b 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", - description: "Retrieves the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouse/get-config)", + description: "Retrieves the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 331b3157ef15c..ebdad161fe477 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", - description: "Retrieves the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/get-permissions)", + description: "Retrieves the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 4fed49f0f8ac5..8e38fb468e43b 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-get-sql-warehouse", name: "Get SQL Warehouse", - description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/get)", + description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/get)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index 28205283bd3e2..00dfe83934543 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-list-sql-warehouses", name: "List SQL Warehouses", - description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/list)", + description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/list)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index d7c38fd1094e8..09d98a9bf73bc 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", - description: "Updates the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouse/set-config)", + description: "Updates the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 2f1d664c96b58..3c8d2b17314e6 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-set-sql-warehouse-permissions", name: "Set SQL Warehouse Permissions", - description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouse/set-permissions)", + description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/setpermissions)", version: "0.0.1", type: "action", props: { @@ -13,16 +13,27 @@ export default { label: "Warehouse ID", description: "The ID of the SQL Warehouse to update permissions for", }, - permissions: { - type: "object", - label: "Permissions", - description: "The permissions object. Example: `{ \"access_control_list\": [{ \"user_name\": \"alice@example.com\", \"permission_level\": \"CAN_MANAGE\" }] }`", + accessControlList: { + type: "object[]", + label: "Access Control List", + description: "List of access control entries. Each entry must include one of `user_name`, `group_name`, or `service_principal_name`, and a `permission_level` (`CAN_VIEW`, `CAN_MONITOR`, `CAN_USE`, `CAN_MANAGE`).", + properties: { + user_name: { type: "string", optional: true }, + group_name: { type: "string", optional: true }, + service_principal_name: { type: "string", optional: true }, + permission_level: { + type: "string", + options: ["CAN_VIEW", "CAN_MONITOR", "CAN_USE", "CAN_MANAGE"] + }, + }, }, }, async run({ $ }) { const response = await this.databricks.setSQLWarehousePermissions({ warehouseId: this.warehouseId, - data: this.permissions, + data: { + access_control_list: this.accessControlList, + }, $, }); diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 1765702cc5fee..95033ca2aebf4 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-start-sql-warehouse", name: "Start SQL Warehouse", - description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/start)", + description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/start)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index 58f227c2e8fdd..d3dcf4da2eeca 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-stop-sql-warehouse", name: "Stop SQL Warehouse", - description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouse/stop)", + description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/stop)", version: "0.0.1", type: "action", props: { From 9292e93dd8d2c7d9df36562f53935bb06fef7ada Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Thu, 28 Aug 2025 21:48:24 +0530 Subject: [PATCH 03/39] fix(databricks): bump component versions and apply lint fixes --- .../delete-sql-warehouse.mjs | 4 ++- .../actions/get-run-output/get-run-output.mjs | 2 +- .../get-sql-warehouse-config.mjs | 4 ++- .../actions/list-runs/list-runs.mjs | 2 +- .../list-sql-warehouses.mjs | 4 ++- .../actions/run-job-now/run-job-now.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 26 +++++++++++---- components/databricks/databricks.app.mjs | 32 +++++++++++++------ components/databricks/package.json | 2 +- 9 files changed, 56 insertions(+), 22 deletions(-) diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 87278c82a46ee..acb9a1e079801 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -21,6 +21,8 @@ export default { }); $.export("$summary", `Successfully deleted SQL Warehouse with ID ${this.warehouseId}`); - return { success: true }; + return { + success: true, + }; }, }; diff --git a/components/databricks/actions/get-run-output/get-run-output.mjs b/components/databricks/actions/get-run-output/get-run-output.mjs index 1fb6c77c9fcdd..ecfa86fcf0576 100644 --- a/components/databricks/actions/get-run-output/get-run-output.mjs +++ b/components/databricks/actions/get-run-output/get-run-output.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run-output", name: "Get Run Output", description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index 9b060a41ec54b..f84cc9996d4a3 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -10,7 +10,9 @@ export default { databricks, }, async run({ $ }) { - const response = await this.databricks.getSQLWarehouseConfig({ $ }); + const response = await this.databricks.getSQLWarehouseConfig({ + $, + }); $.export("$summary", "Successfully retrieved SQL Warehouse configuration"); return response; }, diff --git a/components/databricks/actions/list-runs/list-runs.mjs b/components/databricks/actions/list-runs/list-runs.mjs index 4ad98bd866b54..f24e52369b6e9 100644 --- a/components/databricks/actions/list-runs/list-runs.mjs +++ b/components/databricks/actions/list-runs/list-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-runs", name: "List Runs", description: "Lists all runs available to the user. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index 00dfe83934543..5fc3c39d970d9 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -31,7 +31,9 @@ export default { return []; } - $.export("$summary", `Successfully retrieved ${warehouses.length} warehouse${warehouses.length === 1 ? "" : "s"}.`); + $.export("$summary", `Successfully retrieved ${warehouses.length} warehouse${warehouses.length === 1 + ? "" + : "s"}.`); return warehouses; }, diff --git a/components/databricks/actions/run-job-now/run-job-now.mjs b/components/databricks/actions/run-job-now/run-job-now.mjs index f0c75c2c1a8cb..5de3e46e4ea59 100644 --- a/components/databricks/actions/run-job-now/run-job-now.mjs +++ b/components/databricks/actions/run-job-now/run-job-now.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-run-job-now", name: "Run Job Now", description: "Run a job now and return the id of the triggered run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 3c8d2b17314e6..dd8ac202c7c28 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -18,12 +18,26 @@ export default { label: "Access Control List", description: "List of access control entries. Each entry must include one of `user_name`, `group_name`, or `service_principal_name`, and a `permission_level` (`CAN_VIEW`, `CAN_MONITOR`, `CAN_USE`, `CAN_MANAGE`).", properties: { - user_name: { type: "string", optional: true }, - group_name: { type: "string", optional: true }, - service_principal_name: { type: "string", optional: true }, - permission_level: { - type: "string", - options: ["CAN_VIEW", "CAN_MONITOR", "CAN_USE", "CAN_MANAGE"] + user_name: { + type: "string", + optional: true, + }, + group_name: { + type: "string", + optional: true, + }, + service_principal_name: { + type: "string", + optional: true, + }, + permission_level: { + type: "string", + options: [ + "CAN_VIEW", + "CAN_MONITOR", + "CAN_USE", + "CAN_MANAGE", + ], }, }, }, diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 76420a0d5bdc6..f82c6c311b89e 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -97,14 +97,18 @@ export default { ...args, }); }, - deleteSQLWarehouse({ warehouseId, ...args }) { + deleteSQLWarehouse({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/sql/warehouses/${warehouseId}`, method: "DELETE", ...args, }); }, - getSQLWarehouse({ warehouseId, ...args }) { + getSQLWarehouse({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/sql/warehouses/${warehouseId}`, method: "GET", @@ -117,14 +121,18 @@ export default { ...args, }); }, - editSQLWarehouse({ warehouseId, ...args }) { + editSQLWarehouse({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/sql/warehouses/${warehouseId}/edit`, method: "POST", ...args, }); }, - startSQLWarehouse({ warehouseId, ...args }) { + startSQLWarehouse({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/sql/warehouses/${warehouseId}/start`, method: "POST", @@ -132,14 +140,16 @@ export default { }); }, - stopSQLWarehouse({ warehouseId, ...args }) { + stopSQLWarehouse({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/sql/warehouses/${warehouseId}/stop`, method: "POST", ...args, }); }, - + getSQLWarehouseConfig(args = {}) { return this._makeRequest({ path: "/sql/config/warehouses", @@ -155,7 +165,9 @@ export default { ...args, }); }, - getSQLWarehousePermissions({ warehouseId, ...args }) { + getSQLWarehousePermissions({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/permissions/warehouses/${warehouseId}`, method: "GET", @@ -163,12 +175,14 @@ export default { }); }, - setSQLWarehousePermissions({ warehouseId, ...args }) { + setSQLWarehousePermissions({ + warehouseId, ...args + }) { return this._makeRequest({ path: `/permissions/warehouses/${warehouseId}`, method: "PUT", ...args, }); - } + }, }, }; diff --git a/components/databricks/package.json b/components/databricks/package.json index 50b863968a934..deb7204116952 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.1.0", + "version": "0.1.1", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [ From 6a9646c0473e61271740b4dd292d00e507237bc7 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Fri, 29 Aug 2025 21:43:50 +0530 Subject: [PATCH 04/39] fix(databricks): addressed requested changes --- .../create-sql-warehouse.mjs | 5 + .../delete-sql-warehouse.mjs | 6 +- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 124 ++++++++++++++- .../get-sql-warehouse-permissions.mjs | 6 +- .../get-sql-warehouse/get-sql-warehouse.mjs | 6 +- .../list-sql-warehouses.mjs | 12 -- .../set-sql-warehouse-config.mjs | 148 +++++++++++++++++- .../set-sql-warehouse-permissions.mjs | 44 ++---- .../start-sql-warehouse.mjs | 6 +- .../stop-sql-warehouse/stop-sql-warehouse.mjs | 6 +- components/databricks/databricks.app.mjs | 15 ++ components/databricks/package.json | 2 +- 12 files changed, 320 insertions(+), 60 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 3cb6c11f6ad51..f8d94dc8ba4ba 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -17,6 +17,11 @@ export default { type: "string", label: "Cluster Size", description: "Size of the cluster (e.g., `Small`, `Medium`, `Large`)", + options: [ + "Small", + "Medium", + "Large", + ], }, autoStopMinutes: { type: "integer", diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index acb9a1e079801..2eeea69a7713a 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -9,9 +9,11 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to delete", + propDefinition: [ + databricks, + "warehouseId", + ], }, }, async run({ $ }) { diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 4cf5accf2493b..4eb1b5b3719ad 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -9,20 +9,132 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to edit", + propDefinition: [ + databricks, + "warehouseId", + ], + }, + name: { + type: "string", + label: "Warehouse Name", + description: "Logical name for the warehouse. Must be unique within an org and under 100 characters.", + optional: true, + }, + clusterSize: { + type: "string", + label: "Cluster Size", + description: "Size of clusters allocated for this warehouse.", + options: [ + "2X-Small", + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", + ], + optional: true, + }, + autoStopMins: { + type: "integer", + label: "Auto Stop (minutes)", + description: "Minutes of inactivity before auto-stop. 0 disables autostop. Must be 0 or ≥ 10.", + optional: true, + }, + minNumClusters: { + type: "integer", + label: "Min Number of Clusters", + description: "Minimum number of available clusters (> 0 and ≤ min(max_num_clusters, 30)).", + optional: true, + }, + maxNumClusters: { + type: "integer", + label: "Max Number of Clusters", + description: "Maximum number of clusters for autoscaler (≥ min_num_clusters and ≤ 30).", + optional: true, + }, + enablePhoton: { + type: "boolean", + label: "Enable Photon", + description: "Use Photon optimized clusters.", + optional: true, + }, + enableServerlessCompute: { + type: "boolean", + label: "Enable Serverless Compute", + description: "Use serverless compute for this warehouse.", + optional: true, }, - body: { + warehouseType: { + type: "string", + label: "Warehouse Type", + description: "Set to PRO (recommended) or CLASSIC. Set PRO + enable serverless to use serverless.", + options: [ + "TYPE_UNSPECIFIED", + "CLASSIC", + "PRO", + ], + optional: true, + }, + spotInstancePolicy: { + type: "string", + label: "Spot Instance Policy", + description: "Whether the warehouse should use spot instances.", + options: [ + "POLICY_UNSPECIFIED", + "COST_OPTIMIZED", + "RELIABILITY_OPTIMIZED", + ], + optional: true, + }, + tags: { + type: "object", + label: "Tags", + description: "Key-value tags for all resources associated with this warehouse (fewer than 45 tags).", + optional: true, + }, + channel: { type: "object", - label: "Edit Configuration", - description: "The new configuration for the SQL Warehouse (JSON object). Example: `{ \"name\": \"Updated Warehouse\", \"cluster_size\": \"2X-Small\" }`", + label: "Channel", + description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CURRENT\", \"dbsql_version\": \"2023.35\" }`", + optional: true, + }, + creatorName: { + type: "string", + label: "Creator Name", + description: "Warehouse creator name.", + optional: true, + }, + instanceProfileArn: { + type: "string", + label: "Instance Profile ARN (Deprecated)", + description: "Deprecated. Instance profile used to pass IAM role to the cluster.", + optional: true, }, }, async run({ $ }) { + const payload = {}; + if (this.name) payload.name = this.name; + if (this.clusterSize) payload.cluster_size = this.clusterSize; + if (this.autoStopMins) payload.auto_stop_mins = this.autoStopMins; + if (this.minNumClusters) payload.min_num_clusters = this.minNumClusters; + if (this.maxNumClusters) payload.max_num_clusters = this.maxNumClusters; + if (this.enablePhoton) payload.enable_photon = this.enablePhoton; + if (this.enableServerlessCompute){ + payload.enable_serverless_compute = this.enableServerlessCompute; + } + if (this.warehouseType) payload.warehouse_type = this.warehouseType; + if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; + if (this.tags) payload.tags = this.tags; + if (this.channel) payload.channel = this.channel; + if (this.creatorName) payload.creator_name = this.creatorName; + if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; const response = await this.databricks.editSQLWarehouse({ warehouseId: this.warehouseId, - data: this.body, + data: payload, $, }); diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index ebdad161fe477..852a416f748f0 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -9,9 +9,11 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to fetch permissions for", + propDefinition: [ + databricks, + "warehouseId", + ], }, }, async run({ $ }) { diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 8e38fb468e43b..3c5849846d41f 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -9,9 +9,11 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to retrieve", + propDefinition: [ + databricks, + "warehouseId", + ], }, }, async run({ $ }) { diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index 5fc3c39d970d9..fbcf1865e22ed 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -8,21 +8,9 @@ export default { type: "action", props: { databricks, - maxResults: { - type: "integer", - label: "Max Results", - description: "Maximum number of warehouses to return", - default: 50, - optional: true, - }, }, async run({ $ }) { - const params = { - limit: this.maxResults || 50, - }; - const { warehouses } = await this.databricks.listSQLWarehouses({ - params, $, }); diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 09d98a9bf73bc..8ad1f47a4bc4d 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -8,15 +8,155 @@ export default { type: "action", props: { databricks, - config: { + instanceProfileArn: { + type: "string", + label: "Instance Profile ARN", + description: "Instance profile ARN used to pass an IAM role to clusters (AWS).", + optional: true, + }, + googleServiceAccount: { + type: "string", + label: "Google Service Account", + description: "Service account email used for GCP workspaces, if applicable.", + optional: true, + }, + securityPolicy: { + type: "string", + label: "Security Policy", + description: "Workspace-wide security policy for SQL Warehouses (if applicable).", + options: [ + "NONE", + "DATA_ACCESS_CONTROL", + "PASSTHROUGH", + ], + optional: true, + }, + channel: { type: "object", - label: "Configuration", - description: "The configuration object for SQL Warehouses. Example: `{ \"enable_serverless_compute\": true }`", + label: "Channel", + description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_PREVIEW\", \"dbsql_version\": \"2023.35\" }`", + optional: true, + }, + enabledWarehouseTypes: { + type: "object[]", + label: "Enabled Warehouse Types", + description: "Specify which warehouse types are enabled. Example item: `{ \"warehouse_type\": \"PRO\", \"enabled\": true }`", + properties: { + warehouse_type: { + type: "string", + label: "Warehouse Type", + options: [ + "TYPE_UNSPECIFIED", + "CLASSIC", + "PRO", + ], + }, + enabled: { + type: "boolean", + label: "Enabled", + }, + }, + optional: true, + }, + configParam: { + type: "object[]", + label: "Config Parameters", + description: "General config key/value pairs. Example item: `{ \"key\": \"some.config\", \"value\": \"true\" }`", + properties: { + key: { + type: "string", + label: "Key", + }, + value: { + type: "string", + label: "Value", + }, + }, + optional: true, + }, + globalParam: { + type: "object[]", + label: "Global Parameters", + description: "Global config key/value pairs applied to all warehouses.", + properties: { + key: { + type: "string", + label: "Key", + }, + value: { + type: "string", + label: "Value", + }, + }, + optional: true, + }, + sqlConfigurationParameters: { + type: "object[]", + label: "SQL Configuration Parameters", + description: "SQL-specific configuration key/value pairs.", + properties: { + key: { + type: "string", + label: "Key", + }, + value: { + type: "string", + label: "Value", + }, + }, + optional: true, + }, + dataAccessConfig: { + type: "object[]", + label: "Data Access Config", + description: "Key/value pairs for data access configuration (e.g., credentials passthrough, external storage access).", + properties: { + key: { + type: "string", + label: "Key", + }, + value: { + type: "string", + label: "Value", + }, + }, + optional: true, }, }, async run({ $ }) { + const payload = {}; + + if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; + if (this.googleServiceAccount) payload.google_service_account = this.googleServiceAccount; + if (this.securityPolicy) payload.security_policy = this.securityPolicy; + if (this.channel) payload.channel = this.channel; + + if (Array.isArray(this.enabledWarehouseTypes) && this.enabledWarehouseTypes.length) { + payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item) => ({ + warehouse_type: item.warehouse_type, + enabled: Boolean(item.enabled), + })); + } + if (Array.isArray(this.configParam) && this.configParam.length) { + payload.config_param = { + configuration_pairs: this.configParam, + }; + } + if (Array.isArray(this.globalParam) && this.globalParam.length) { + payload.global_param = { + configuration_pairs: this.globalParam, + }; + } + if (Array.isArray(this.sqlConfigurationParameters) && this.sqlConfigurationParameters.length) { + payload.sql_configuration_parameters = { + configuration_pairs: this.sqlConfigurationParameters, + }; + } + if (Array.isArray(this.dataAccessConfig) && this.dataAccessConfig.length) { + payload.data_access_config = this.dataAccessConfig; + } const response = await this.databricks.setSQLWarehouseConfig({ - data: this.config, + data: payload, $, }); diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index dd8ac202c7c28..be87b485254f3 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -9,44 +9,33 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to update permissions for", + propDefinition: [ + databricks, + "warehouseId", + ], }, accessControlList: { - type: "object[]", + type: "string[]", label: "Access Control List", description: "List of access control entries. Each entry must include one of `user_name`, `group_name`, or `service_principal_name`, and a `permission_level` (`CAN_VIEW`, `CAN_MONITOR`, `CAN_USE`, `CAN_MANAGE`).", - properties: { - user_name: { - type: "string", - optional: true, - }, - group_name: { - type: "string", - optional: true, - }, - service_principal_name: { - type: "string", - optional: true, - }, - permission_level: { - type: "string", - options: [ - "CAN_VIEW", - "CAN_MONITOR", - "CAN_USE", - "CAN_MANAGE", - ], - }, - }, }, }, async run({ $ }) { + let acl = []; + try { + acl = (this.accessControlList || []).map((entry) => + typeof entry === "string" + ? JSON.parse(entry) + : entry); + } catch (err) { + throw new Error(`Invalid JSON in Access Control List: ${err.message}`); + } + const response = await this.databricks.setSQLWarehousePermissions({ warehouseId: this.warehouseId, data: { - access_control_list: this.accessControlList, + access_control_list: acl, }, $, }); @@ -54,4 +43,5 @@ export default { $.export("$summary", `Successfully updated permissions for SQL Warehouse ID ${this.warehouseId}`); return response; }, + }; diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 95033ca2aebf4..2de2b61cba5ac 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -9,9 +9,11 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to start", + propDefinition: [ + databricks, + "warehouseId", + ], }, }, async run({ $ }) { diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index d3dcf4da2eeca..1388ec5102b61 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -9,9 +9,11 @@ export default { props: { databricks, warehouseId: { - type: "string", - label: "Warehouse ID", description: "The ID of the SQL Warehouse to stop", + propDefinition: [ + databricks, + "warehouseId", + ], }, }, async run({ $ }) { diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index f82c6c311b89e..05d6f88d697ab 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -44,6 +44,21 @@ export default { })) || []; }, }, + warehouseId: { + type: "string", + label: "Warehouse ID", + description: "The ID of the SQL Warehouse to get runs from", + async options() { + const { warehouses } = await this.listSQLWarehouses(); + return warehouses?.map(({ + id: value, + name: label, + }) => ({ + value, + label, + })) || []; + }, + }, }, methods: { _baseUrl() { diff --git a/components/databricks/package.json b/components/databricks/package.json index deb7204116952..f49e7482191ef 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.1.1", + "version": "0.2.0", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [ From d66788ba10e68efa028089f33dc8a55a2d0d2bbb Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Fri, 29 Aug 2025 22:30:10 +0530 Subject: [PATCH 05/39] addressed coderabbit review feedback --- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 4eb1b5b3719ad..824315367d3b8 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -117,21 +117,48 @@ export default { }, async run({ $ }) { const payload = {}; + if (this.name) payload.name = this.name; if (this.clusterSize) payload.cluster_size = this.clusterSize; - if (this.autoStopMins) payload.auto_stop_mins = this.autoStopMins; - if (this.minNumClusters) payload.min_num_clusters = this.minNumClusters; - if (this.maxNumClusters) payload.max_num_clusters = this.maxNumClusters; - if (this.enablePhoton) payload.enable_photon = this.enablePhoton; - if (this.enableServerlessCompute){ + + if (this.autoStopMins !== undefined) payload.auto_stop_mins = this.autoStopMins; + if (this.minNumClusters !== undefined) payload.min_num_clusters = this.minNumClusters; + if (this.maxNumClusters !== undefined) payload.max_num_clusters = this.maxNumClusters; + + if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton; + if (this.enableServerlessCompute !== undefined) { payload.enable_serverless_compute = this.enableServerlessCompute; } + if (this.warehouseType) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; + if (this.tags) payload.tags = this.tags; if (this.channel) payload.channel = this.channel; if (this.creatorName) payload.creator_name = this.creatorName; if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; + + if (!Object.keys(payload).length) { + throw new Error("No fields to update. Provide at least one property."); + } + if (this.autoStopMins !== undefined && this.autoStopMins !== 0 && this.autoStopMins < 10) { + throw new Error("autoStopMins must be 0 or >= 10."); + } + if (this.minNumClusters !== undefined && (this.minNumClusters < 1 || this.minNumClusters > 30)) { + throw new Error("minNumClusters must be between 1 and 30."); + } + if (this.maxNumClusters !== undefined && (this.maxNumClusters < 1 || this.maxNumClusters > 30)) { + throw new Error("maxNumClusters must be between 1 and 30."); + } + if (this.minNumClusters !== undefined && + this.maxNumClusters !== undefined && + this.minNumClusters > this.maxNumClusters) { + throw new Error("minNumClusters must be <= maxNumClusters."); + } + if (this.enableServerlessCompute === true && this.warehouseType === "CLASSIC") { + throw new Error("Serverless compute requires warehouseType = PRO."); + } + const response = await this.databricks.editSQLWarehouse({ warehouseId: this.warehouseId, data: payload, @@ -140,5 +167,6 @@ export default { $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); return response; - }, + } + }; From e1205881f05fee0cfc7c4e9d7a9b688946a5e7bc Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Fri, 29 Aug 2025 23:02:54 +0530 Subject: [PATCH 06/39] resolved the linting issues --- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 824315367d3b8..18b6296fd224d 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -144,10 +144,16 @@ export default { if (this.autoStopMins !== undefined && this.autoStopMins !== 0 && this.autoStopMins < 10) { throw new Error("autoStopMins must be 0 or >= 10."); } - if (this.minNumClusters !== undefined && (this.minNumClusters < 1 || this.minNumClusters > 30)) { + if ( + this.minNumClusters !== undefined && + (this.minNumClusters < 1 || this.minNumClusters > 30) + ) { throw new Error("minNumClusters must be between 1 and 30."); } - if (this.maxNumClusters !== undefined && (this.maxNumClusters < 1 || this.maxNumClusters > 30)) { + if ( + this.maxNumClusters !== undefined && + (this.maxNumClusters < 1 || this.maxNumClusters > 30) + ) { throw new Error("maxNumClusters must be between 1 and 30."); } if (this.minNumClusters !== undefined && @@ -167,6 +173,6 @@ export default { $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); return response; - } + }, }; From e742ec283a5ea04dbc497d61e3c6ccd07b5d10d0 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 15:12:16 +0530 Subject: [PATCH 07/39] addressed all test failures --- .../create-sql-warehouse.mjs | 134 +++++++++++++++++- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 63 ++++---- .../set-sql-warehouse-config.mjs | 29 ++-- .../set-sql-warehouse-permissions.mjs | 10 +- 4 files changed, 173 insertions(+), 63 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index f8d94dc8ba4ba..147aa530209bf 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -16,34 +16,160 @@ export default { clusterSize: { type: "string", label: "Cluster Size", - description: "Size of the cluster (e.g., `Small`, `Medium`, `Large`)", + description: "Size of the cluster", options: [ + "2X-Small", + "X-Small", "Small", "Medium", "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", ], }, autoStopMinutes: { type: "integer", label: "Auto Stop (minutes)", - description: "Number of minutes of inactivity before the warehouse auto-stops", + description: + "Minutes of inactivity before auto-stop. 0 disables auto-stop. Must be 0 or ≥ 10.", optional: true, default: 10, }, + minNumClusters: { + type: "integer", + label: "Min Number of Clusters", + description: "Minimum number of clusters to maintain (> 0 and ≤ min(max_num_clusters, 30)).", + optional: true, + default: 1, + }, + maxNumClusters: { + type: "integer", + label: "Max Number of Clusters", + description: "Maximum number of clusters for autoscaler (≥ min_num_clusters and ≤ 30).", + optional: true, + }, + enablePhoton: { + type: "boolean", + label: "Enable Photon", + description: "Whether the warehouse should use Photon optimized clusters.", + optional: true, + }, + enableServerlessCompute: { + type: "boolean", + label: "Enable Serverless Compute", + description: "Whether the warehouse should use serverless compute.", + optional: true, + }, + warehouseType: { + type: "string", + label: "Warehouse Type", + description: + "Warehouse type: PRO or CLASSIC. Set PRO + enableServerlessCompute = true to use serverless.", + options: [ + "TYPE_UNSPECIFIED", + "CLASSIC", + "PRO", + ], + optional: true, + }, + spotInstancePolicy: { + type: "string", + label: "Spot Instance Policy", + description: "Configures whether the warehouse should use spot instances.", + options: [ + "POLICY_UNSPECIFIED", + "COST_OPTIMIZED", + "RELIABILITY_OPTIMIZED", + ], + optional: true, + }, + channel: { + type: "object", + label: "Channel", + description: + "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CURRENT\", \"dbsql_version\": \"2023.35\" }`", + optional: true, + }, + tags: { + type: "object[]", + label: "Tags", + description: + "Custom key-value tags for resources associated with this SQL Warehouse.", + properties: { + key: { + type: "string", + label: "Key", + }, + value: { + type: "string", + label: "Value", + }, + }, + optional: true, + }, + instanceProfileArn: { + type: "string", + label: "Instance Profile ARN (Deprecated)", + description: "Deprecated. Instance profile used to pass IAM role to the cluster.", + optional: true, + }, }, + async run({ $ }) { const payload = { name: this.name, cluster_size: this.clusterSize, - auto_stop_mins: this.autoStopMinutes, }; + if (this.autoStopMinutes !== undefined) { + if (this.autoStopMinutes !== 0 && this.autoStopMinutes < 10) { + throw new Error("autoStopMinutes must be 0 or ≥ 10."); + } + payload.auto_stop_mins = this.autoStopMinutes; + } + + payload.min_num_clusters = this.minNumClusters || 1; + if (payload.min_num_clusters < 1) { + throw new Error("minNumClusters must be ≥ 1."); + } + + if (this.maxNumClusters !== undefined) { + if ( + this.maxNumClusters < payload.min_num_clusters || + this.maxNumClusters > 30 + ) { + throw new Error( + `maxNumClusters must be ≥ minNumClusters (${payload.min_num_clusters}) and ≤ 30.`, + ); + } + payload.max_num_clusters = this.maxNumClusters; + } + + if (this.enablePhoton !== undefined) + payload.enable_photon = this.enablePhoton; + if (this.enableServerlessCompute !== undefined) + payload.enable_serverless_compute = this.enableServerlessCompute; + if (this.warehouseType) payload.warehouse_type = this.warehouseType; + if (this.spotInstancePolicy) + payload.spot_instance_policy = this.spotInstancePolicy; + if (this.channel) payload.channel = this.channel; + if (this.tags?.length) payload.tags = { + custom_tags: this.tags, + }; + if (this.instanceProfileArn) + payload.instance_profile_arn = this.instanceProfileArn; + const response = await this.databricks.createSQLWarehouse({ data: payload, $, }); - $.export("$summary", `Successfully created SQL Warehouse: ${response?.name || this.name}`); + $.export( + "$summary", + `Successfully created SQL Warehouse: ${response?.name || this.name}`, + ); return response; }, }; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 18b6296fd224d..04827f917431b 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -102,12 +102,6 @@ export default { description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CURRENT\", \"dbsql_version\": \"2023.35\" }`", optional: true, }, - creatorName: { - type: "string", - label: "Creator Name", - description: "Warehouse creator name.", - optional: true, - }, instanceProfileArn: { type: "string", label: "Instance Profile ARN (Deprecated)", @@ -121,49 +115,47 @@ export default { if (this.name) payload.name = this.name; if (this.clusterSize) payload.cluster_size = this.clusterSize; - if (this.autoStopMins !== undefined) payload.auto_stop_mins = this.autoStopMins; - if (this.minNumClusters !== undefined) payload.min_num_clusters = this.minNumClusters; - if (this.maxNumClusters !== undefined) payload.max_num_clusters = this.maxNumClusters; + if (this.autoStopMins !== undefined) { + if (this.autoStopMins !== 0 && this.autoStopMins < 10) { + throw new Error("autoStopMins must be 0 or >= 10."); + } + payload.auto_stop_mins = this.autoStopMins; + } + + if (this.minNumClusters !== undefined) { + if (this.minNumClusters < 1 || this.minNumClusters > 30) { + throw new Error("minNumClusters must be between 1 and 30."); + } + payload.min_num_clusters = this.minNumClusters; + } + + if (this.maxNumClusters !== undefined) { + if (this.maxNumClusters < 1 || this.maxNumClusters > 30) { + throw new Error("maxNumClusters must be between 1 and 30."); + } + if (this.minNumClusters !== undefined && this.maxNumClusters < this.minNumClusters) { + throw new Error("maxNumClusters must be >= minNumClusters."); + } + payload.max_num_clusters = this.maxNumClusters; + } if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton; if (this.enableServerlessCompute !== undefined) { + if (this.warehouseType === "CLASSIC" && this.enableServerlessCompute) { + throw new Error("Serverless compute requires warehouseType = PRO."); + } payload.enable_serverless_compute = this.enableServerlessCompute; } if (this.warehouseType) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; - if (this.tags) payload.tags = this.tags; if (this.channel) payload.channel = this.channel; - if (this.creatorName) payload.creator_name = this.creatorName; if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; if (!Object.keys(payload).length) { throw new Error("No fields to update. Provide at least one property."); } - if (this.autoStopMins !== undefined && this.autoStopMins !== 0 && this.autoStopMins < 10) { - throw new Error("autoStopMins must be 0 or >= 10."); - } - if ( - this.minNumClusters !== undefined && - (this.minNumClusters < 1 || this.minNumClusters > 30) - ) { - throw new Error("minNumClusters must be between 1 and 30."); - } - if ( - this.maxNumClusters !== undefined && - (this.maxNumClusters < 1 || this.maxNumClusters > 30) - ) { - throw new Error("maxNumClusters must be between 1 and 30."); - } - if (this.minNumClusters !== undefined && - this.maxNumClusters !== undefined && - this.minNumClusters > this.maxNumClusters) { - throw new Error("minNumClusters must be <= maxNumClusters."); - } - if (this.enableServerlessCompute === true && this.warehouseType === "CLASSIC") { - throw new Error("Serverless compute requires warehouseType = PRO."); - } const response = await this.databricks.editSQLWarehouse({ warehouseId: this.warehouseId, @@ -173,6 +165,7 @@ export default { $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); return response; - }, + } +, }; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 8ad1f47a4bc4d..bbc62366cf033 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -38,24 +38,9 @@ export default { optional: true, }, enabledWarehouseTypes: { - type: "object[]", + type: "string[]", label: "Enabled Warehouse Types", description: "Specify which warehouse types are enabled. Example item: `{ \"warehouse_type\": \"PRO\", \"enabled\": true }`", - properties: { - warehouse_type: { - type: "string", - label: "Warehouse Type", - options: [ - "TYPE_UNSPECIFIED", - "CLASSIC", - "PRO", - ], - }, - enabled: { - type: "boolean", - label: "Enabled", - }, - }, optional: true, }, configParam: { @@ -132,10 +117,14 @@ export default { if (this.channel) payload.channel = this.channel; if (Array.isArray(this.enabledWarehouseTypes) && this.enabledWarehouseTypes.length) { - payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item) => ({ - warehouse_type: item.warehouse_type, - enabled: Boolean(item.enabled), - })); + try { + payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item) => + typeof item === "string" + ? JSON.parse(item) + : item); + } catch (err) { + throw new Error(`Invalid JSON in enabledWarehouseTypes: ${err.message}`); + } } if (Array.isArray(this.configParam) && this.configParam.length) { payload.config_param = { diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index be87b485254f3..09707eb43e9e6 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -24,10 +24,12 @@ export default { async run({ $ }) { let acl = []; try { - acl = (this.accessControlList || []).map((entry) => - typeof entry === "string" - ? JSON.parse(entry) - : entry); + acl = (this.accessControlList || []) + .map((entry) => + typeof entry === "string" + ? JSON.parse(entry) + : entry) + .filter((entry) => entry && Object.keys(entry).length > 0); } catch (err) { throw new Error(`Invalid JSON in Access Control List: ${err.message}`); } From 01ed5097516be7640775ab532a6df89c0ff9dc83 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 16:01:04 +0530 Subject: [PATCH 08/39] addressed coderabbit review feedback --- .../create-sql-warehouse.mjs | 7 +-- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 19 +++++--- .../set-sql-warehouse-config.mjs | 44 +++++++++---------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 147aa530209bf..99372ecb777d0 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -130,10 +130,11 @@ export default { payload.auto_stop_mins = this.autoStopMinutes; } - payload.min_num_clusters = this.minNumClusters || 1; - if (payload.min_num_clusters < 1) { - throw new Error("minNumClusters must be ≥ 1."); + const minNumClusters = this.minNumClusters ?? 1; + if (minNumClusters < 1 || minNumClusters > 30) { + throw new Error("minNumClusters must be between 1 and 30."); } + payload.min_num_clusters = minNumClusters; if (this.maxNumClusters !== undefined) { if ( diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 04827f917431b..0c55cda91a967 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -112,8 +112,13 @@ export default { async run({ $ }) { const payload = {}; - if (this.name) payload.name = this.name; - if (this.clusterSize) payload.cluster_size = this.clusterSize; + if (this.name !== undefined) { + if (typeof this.name !== "string" || this.name.length >= 100) { + throw new Error("name must be a string with fewer than 100 characters."); + } + payload.name = this.name; + } + if (this.clusterSize !== undefined) payload.cluster_size = this.clusterSize; if (this.autoStopMins !== undefined) { if (this.autoStopMins !== 0 && this.autoStopMins < 10) { @@ -147,11 +152,11 @@ export default { payload.enable_serverless_compute = this.enableServerlessCompute; } - if (this.warehouseType) payload.warehouse_type = this.warehouseType; - if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; - if (this.tags) payload.tags = this.tags; - if (this.channel) payload.channel = this.channel; - if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; + if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType; + if (this.spotInstancePolicy !== undefined) payload.spot_instance_policy = this.spotInstancePolicy; + if (this.tags !== undefined) payload.tags = this.tags; + if (this.channel !== undefined) payload.channel = this.channel; + if (this.instanceProfileArn !== undefined) payload.instance_profile_arn = this.instanceProfileArn; if (!Object.keys(payload).length) { throw new Error("No fields to update. Provide at least one property."); diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index bbc62366cf033..7924ac6b35d50 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -109,37 +109,35 @@ export default { }, }, async run({ $ }) { - const payload = {}; - - if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; - if (this.googleServiceAccount) payload.google_service_account = this.googleServiceAccount; - if (this.securityPolicy) payload.security_policy = this.securityPolicy; - if (this.channel) payload.channel = this.channel; + const current = await this.databricks.getSQLWarehouseConfig({ $ }); + const payload = { ...current }; + if (typeof this.enableServerlessCompute === "boolean") { + payload.enable_serverless_compute = this.enableServerlessCompute; + } + if (this.instanceProfileArn !== undefined) { + payload.instance_profile_arn = this.instanceProfileArn; + } + if (this.googleServiceAccount !== undefined) { + payload.google_service_account = this.googleServiceAccount; + } + if (this.securityPolicy !== undefined) { + payload.security_policy = this.securityPolicy; + } + if (this.channel !== undefined) { + payload.channel = this.channel; + } if (Array.isArray(this.enabledWarehouseTypes) && this.enabledWarehouseTypes.length) { - try { - payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item) => - typeof item === "string" - ? JSON.parse(item) - : item); - } catch (err) { - throw new Error(`Invalid JSON in enabledWarehouseTypes: ${err.message}`); - } + payload.enabled_warehouse_types = this.enabledWarehouseTypes; } if (Array.isArray(this.configParam) && this.configParam.length) { - payload.config_param = { - configuration_pairs: this.configParam, - }; + payload.config_param = { configuration_pairs: this.configParam }; } if (Array.isArray(this.globalParam) && this.globalParam.length) { - payload.global_param = { - configuration_pairs: this.globalParam, - }; + payload.global_param = { configuration_pairs: this.globalParam }; } if (Array.isArray(this.sqlConfigurationParameters) && this.sqlConfigurationParameters.length) { - payload.sql_configuration_parameters = { - configuration_pairs: this.sqlConfigurationParameters, - }; + payload.sql_configuration_parameters = { configuration_pairs: this.sqlConfigurationParameters }; } if (Array.isArray(this.dataAccessConfig) && this.dataAccessConfig.length) { payload.data_access_config = this.dataAccessConfig; From 49e997c12daedf7471d19f4d5c7b1e5b021520c1 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 16:13:04 +0530 Subject: [PATCH 09/39] resolved the linting issues --- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 10 +++++++--- .../set-sql-warehouse-config.mjs | 20 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 0c55cda91a967..877c2e5147a38 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -153,10 +153,14 @@ export default { } if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType; - if (this.spotInstancePolicy !== undefined) payload.spot_instance_policy = this.spotInstancePolicy; + if (this.spotInstancePolicy !== undefined) { + payload.spot_instance_policy = this.spotInstancePolicy; + } if (this.tags !== undefined) payload.tags = this.tags; if (this.channel !== undefined) payload.channel = this.channel; - if (this.instanceProfileArn !== undefined) payload.instance_profile_arn = this.instanceProfileArn; + if (this.instanceProfileArn !== undefined) { + payload.instance_profile_arn = this.instanceProfileArn; + } if (!Object.keys(payload).length) { throw new Error("No fields to update. Provide at least one property."); @@ -171,6 +175,6 @@ export default { $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); return response; } -, + , }; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 7924ac6b35d50..40e2515d9fdbf 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -109,8 +109,12 @@ export default { }, }, async run({ $ }) { - const current = await this.databricks.getSQLWarehouseConfig({ $ }); - const payload = { ...current }; + const current = await this.databricks.getSQLWarehouseConfig({ + $, + }); + const payload = { + ...current, + }; if (typeof this.enableServerlessCompute === "boolean") { payload.enable_serverless_compute = this.enableServerlessCompute; @@ -131,13 +135,19 @@ export default { payload.enabled_warehouse_types = this.enabledWarehouseTypes; } if (Array.isArray(this.configParam) && this.configParam.length) { - payload.config_param = { configuration_pairs: this.configParam }; + payload.config_param = { + configuration_pairs: this.configParam, + }; } if (Array.isArray(this.globalParam) && this.globalParam.length) { - payload.global_param = { configuration_pairs: this.globalParam }; + payload.global_param = { + configuration_pairs: this.globalParam, + }; } if (Array.isArray(this.sqlConfigurationParameters) && this.sqlConfigurationParameters.length) { - payload.sql_configuration_parameters = { configuration_pairs: this.sqlConfigurationParameters }; + payload.sql_configuration_parameters = { + configuration_pairs: this.sqlConfigurationParameters, + }; } if (Array.isArray(this.dataAccessConfig) && this.dataAccessConfig.length) { payload.data_access_config = this.dataAccessConfig; From 0535802ae95b7f03b2d9171a74a4d13188fd32b4 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 16:32:28 +0530 Subject: [PATCH 10/39] addressed coderabbit review feedback --- .../set-sql-warehouse-config.mjs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 40e2515d9fdbf..0537d0a673b01 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -132,7 +132,27 @@ export default { payload.channel = this.channel; } if (Array.isArray(this.enabledWarehouseTypes) && this.enabledWarehouseTypes.length) { - payload.enabled_warehouse_types = this.enabledWarehouseTypes; + payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item, idx) => { + let obj = item; + if (typeof item === "string") { + try { obj = JSON.parse(item); } catch (e) { + throw new Error(`enabledWarehouseTypes[${idx}] must be valid JSON: ${e.message}`); + } + } + if (!obj || typeof obj !== "object") { + throw new Error(`enabledWarehouseTypes[${idx}] must be an object with { "warehouse_type": string, "enabled": boolean }`); + } + const { + warehouse_type, enabled, + } = obj; + if (typeof warehouse_type !== "string" || typeof enabled !== "boolean") { + throw new Error(`enabledWarehouseTypes[${idx}] invalid shape; expected { "warehouse_type": string, "enabled": boolean }`); + } + return { + warehouse_type, + enabled: Boolean(enabled), + }; + }); } if (Array.isArray(this.configParam) && this.configParam.length) { payload.config_param = { From b98476ca24097bdde146dd1abcf745ffca5773fb Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 17:53:26 +0530 Subject: [PATCH 11/39] addressed coderabbit review feedback --- .../set-sql-warehouse-config.mjs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 0537d0a673b01..5628c746844ce 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -112,13 +112,22 @@ export default { const current = await this.databricks.getSQLWarehouseConfig({ $, }); - const payload = { - ...current, - }; + const allowed = [ + "enable_serverless_compute", + "instance_profile_arn", + "google_service_account", + "security_policy", + "channel", + "enabled_warehouse_types", + "config_param", + "global_param", + "sql_configuration_parameters", + "data_access_config", + ]; + const payload = Object.fromEntries( + Object.entries(current || {}).filter(([k]) => allowed.includes(k)), + ); - if (typeof this.enableServerlessCompute === "boolean") { - payload.enable_serverless_compute = this.enableServerlessCompute; - } if (this.instanceProfileArn !== undefined) { payload.instance_profile_arn = this.instanceProfileArn; } From 2153ac3ccf33e1f61f6aafcb7a0179723701f7ad Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 1 Sep 2025 20:42:51 +0530 Subject: [PATCH 12/39] resolved the linting issues --- .../set-sql-warehouse-config/set-sql-warehouse-config.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 5628c746844ce..177701274c685 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -125,7 +125,9 @@ export default { "data_access_config", ]; const payload = Object.fromEntries( - Object.entries(current || {}).filter(([k]) => allowed.includes(k)), + Object.entries(current || {}).filter(([ + k, + ]) => allowed.includes(k)), ); if (this.instanceProfileArn !== undefined) { From b04a050113b1683884406ade21bb3c0f29aa50d9 Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Mon, 1 Sep 2025 12:02:54 -0400 Subject: [PATCH 13/39] updates --- .../create-sql-warehouse.mjs | 48 +++++------ .../edit-sql-warehouse/edit-sql-warehouse.mjs | 44 +++++----- .../get-sql-warehouse-config.mjs | 2 +- .../get-sql-warehouse-permissions.mjs | 2 +- .../set-sql-warehouse-config.mjs | 85 ++++++------------- components/databricks/common/constants.mjs | 15 ++++ components/databricks/common/utils.mjs | 31 +++++++ 7 files changed, 116 insertions(+), 111 deletions(-) create mode 100644 components/databricks/common/constants.mjs create mode 100644 components/databricks/common/utils.mjs diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 99372ecb777d0..d857beaeaec0d 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -1,4 +1,7 @@ import databricks from "../../databricks.app.mjs"; +import constants from "../../common/constants.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "databricks-create-sql-warehouse", @@ -17,17 +20,7 @@ export default { type: "string", label: "Cluster Size", description: "Size of the cluster", - options: [ - "2X-Small", - "X-Small", - "Small", - "Medium", - "Large", - "X-Large", - "2X-Large", - "3X-Large", - "4X-Large", - ], + options: constants.CLUSTER_SIZES, }, autoStopMinutes: { type: "integer", @@ -93,20 +86,10 @@ export default { optional: true, }, tags: { - type: "object[]", + type: "object", label: "Tags", description: "Custom key-value tags for resources associated with this SQL Warehouse.", - properties: { - key: { - type: "string", - label: "Key", - }, - value: { - type: "string", - label: "Value", - }, - }, optional: true, }, instanceProfileArn: { @@ -125,14 +108,14 @@ export default { if (this.autoStopMinutes !== undefined) { if (this.autoStopMinutes !== 0 && this.autoStopMinutes < 10) { - throw new Error("autoStopMinutes must be 0 or ≥ 10."); + throw new ConfigurationError("autoStopMinutes must be 0 or ≥ 10."); } payload.auto_stop_mins = this.autoStopMinutes; } const minNumClusters = this.minNumClusters ?? 1; if (minNumClusters < 1 || minNumClusters > 30) { - throw new Error("minNumClusters must be between 1 and 30."); + throw new ConfigurationError("minNumClusters must be between 1 and 30."); } payload.min_num_clusters = minNumClusters; @@ -141,13 +124,22 @@ export default { this.maxNumClusters < payload.min_num_clusters || this.maxNumClusters > 30 ) { - throw new Error( + throw new ConfigurationError( `maxNumClusters must be ≥ minNumClusters (${payload.min_num_clusters}) and ≤ 30.`, ); } payload.max_num_clusters = this.maxNumClusters; } + const parsedTags = utils.parseObject(this.tags); + const tags = Object.entries(parsedTags).map(([ + key, + value, + ]) => ({ + key, + value, + })); + if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton; if (this.enableServerlessCompute !== undefined) @@ -155,10 +147,8 @@ export default { if (this.warehouseType) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; - if (this.channel) payload.channel = this.channel; - if (this.tags?.length) payload.tags = { - custom_tags: this.tags, - }; + if (this.channel) payload.channel = utils.parseObject(this.channel); + if (tags.length) payload.tags = tags; if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 877c2e5147a38..20d164dcc7fd7 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -1,9 +1,12 @@ import databricks from "../../databricks.app.mjs"; +import constants from "../../common/constants.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", - description: "Edits the configuration of an existing SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/edit)", + description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)", version: "0.0.1", type: "action", props: { @@ -25,17 +28,7 @@ export default { type: "string", label: "Cluster Size", description: "Size of clusters allocated for this warehouse.", - options: [ - "2X-Small", - "X-Small", - "Small", - "Medium", - "Large", - "X-Large", - "2X-Large", - "3X-Large", - "4X-Large", - ], + options: constants.CLUSTER_SIZES, optional: true, }, autoStopMins: { @@ -114,7 +107,7 @@ export default { if (this.name !== undefined) { if (typeof this.name !== "string" || this.name.length >= 100) { - throw new Error("name must be a string with fewer than 100 characters."); + throw new ConfigurationError("name must be a string with fewer than 100 characters."); } payload.name = this.name; } @@ -122,24 +115,24 @@ export default { if (this.autoStopMins !== undefined) { if (this.autoStopMins !== 0 && this.autoStopMins < 10) { - throw new Error("autoStopMins must be 0 or >= 10."); + throw new ConfigurationError("autoStopMins must be 0 or >= 10."); } payload.auto_stop_mins = this.autoStopMins; } if (this.minNumClusters !== undefined) { if (this.minNumClusters < 1 || this.minNumClusters > 30) { - throw new Error("minNumClusters must be between 1 and 30."); + throw new ConfigurationError("minNumClusters must be between 1 and 30."); } payload.min_num_clusters = this.minNumClusters; } if (this.maxNumClusters !== undefined) { if (this.maxNumClusters < 1 || this.maxNumClusters > 30) { - throw new Error("maxNumClusters must be between 1 and 30."); + throw new ConfigurationError("maxNumClusters must be between 1 and 30."); } if (this.minNumClusters !== undefined && this.maxNumClusters < this.minNumClusters) { - throw new Error("maxNumClusters must be >= minNumClusters."); + throw new ConfigurationError("maxNumClusters must be >= minNumClusters."); } payload.max_num_clusters = this.maxNumClusters; } @@ -147,23 +140,32 @@ export default { if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton; if (this.enableServerlessCompute !== undefined) { if (this.warehouseType === "CLASSIC" && this.enableServerlessCompute) { - throw new Error("Serverless compute requires warehouseType = PRO."); + throw new ConfigurationError("Serverless compute requires warehouseType = PRO."); } payload.enable_serverless_compute = this.enableServerlessCompute; } + const parsedTags = utils.parseObject(this.tags); + const tags = Object.entries(parsedTags).map(([ + key, + value, + ]) => ({ + key, + value, + })); + if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy !== undefined) { payload.spot_instance_policy = this.spotInstancePolicy; } - if (this.tags !== undefined) payload.tags = this.tags; - if (this.channel !== undefined) payload.channel = this.channel; + if (tags.length) payload.tags = tags; + if (this.channel !== undefined) payload.channel = utils.parseObject(this.channel); if (this.instanceProfileArn !== undefined) { payload.instance_profile_arn = this.instanceProfileArn; } if (!Object.keys(payload).length) { - throw new Error("No fields to update. Provide at least one property."); + throw new ConfigurationError("No fields to update. Provide at least one property."); } const response = await this.databricks.editSQLWarehouse({ diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index f84cc9996d4a3..37202334854c5 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", - description: "Retrieves the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", + description: "Retrieves the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 852a416f748f0..66ec10e7d75a7 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", - description: "Retrieves the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", + description: "Retrieves the permissions for a specific SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", version: "0.0.1", type: "action", props: { diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 177701274c685..b8e562a20d34d 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -1,9 +1,11 @@ import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", - description: "Updates the global configuration for SQL Warehouses. [See docs](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", + description: "Updates the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", version: "0.0.1", type: "action", props: { @@ -44,67 +46,27 @@ export default { optional: true, }, configParam: { - type: "object[]", + type: "object", label: "Config Parameters", description: "General config key/value pairs. Example item: `{ \"key\": \"some.config\", \"value\": \"true\" }`", - properties: { - key: { - type: "string", - label: "Key", - }, - value: { - type: "string", - label: "Value", - }, - }, optional: true, }, globalParam: { - type: "object[]", + type: "object", label: "Global Parameters", description: "Global config key/value pairs applied to all warehouses.", - properties: { - key: { - type: "string", - label: "Key", - }, - value: { - type: "string", - label: "Value", - }, - }, optional: true, }, sqlConfigurationParameters: { - type: "object[]", + type: "object", label: "SQL Configuration Parameters", description: "SQL-specific configuration key/value pairs.", - properties: { - key: { - type: "string", - label: "Key", - }, - value: { - type: "string", - label: "Value", - }, - }, optional: true, }, dataAccessConfig: { - type: "object[]", + type: "object", label: "Data Access Config", description: "Key/value pairs for data access configuration (e.g., credentials passthrough, external storage access).", - properties: { - key: { - type: "string", - label: "Key", - }, - value: { - type: "string", - label: "Value", - }, - }, optional: true, }, }, @@ -140,24 +102,25 @@ export default { payload.security_policy = this.securityPolicy; } if (this.channel !== undefined) { - payload.channel = this.channel; + payload.channel = utils.parseObject(this.channel); } - if (Array.isArray(this.enabledWarehouseTypes) && this.enabledWarehouseTypes.length) { - payload.enabled_warehouse_types = this.enabledWarehouseTypes.map((item, idx) => { + const enabledWarehouseTypes = utils.parseObject(this.enabledWarehouseTypes); + if (Array.isArray(enabledWarehouseTypes) && enabledWarehouseTypes.length) { + payload.enabled_warehouse_types = enabledWarehouseTypes.map((item, idx) => { let obj = item; if (typeof item === "string") { try { obj = JSON.parse(item); } catch (e) { - throw new Error(`enabledWarehouseTypes[${idx}] must be valid JSON: ${e.message}`); + throw new ConfigurationError(`enabledWarehouseTypes[${idx}] must be valid JSON: ${e.message}`); } } if (!obj || typeof obj !== "object") { - throw new Error(`enabledWarehouseTypes[${idx}] must be an object with { "warehouse_type": string, "enabled": boolean }`); + throw new ConfigurationError(`enabledWarehouseTypes[${idx}] must be an object with { "warehouse_type": string, "enabled": boolean }`); } const { warehouse_type, enabled, } = obj; if (typeof warehouse_type !== "string" || typeof enabled !== "boolean") { - throw new Error(`enabledWarehouseTypes[${idx}] invalid shape; expected { "warehouse_type": string, "enabled": boolean }`); + throw new ConfigurationError(`enabledWarehouseTypes[${idx}] invalid shape; expected { "warehouse_type": string, "enabled": boolean }`); } return { warehouse_type, @@ -165,23 +128,27 @@ export default { }; }); } - if (Array.isArray(this.configParam) && this.configParam.length) { + const configParam = utils.parseObject(this.configParam); + if (Array.isArray(configParam) && configParam.length) { payload.config_param = { - configuration_pairs: this.configParam, + configuration_pairs: configParam, }; } - if (Array.isArray(this.globalParam) && this.globalParam.length) { + const globalParam = utils.parseObject(this.globalParam); + if (Array.isArray(globalParam) && globalParam.length) { payload.global_param = { - configuration_pairs: this.globalParam, + configuration_pairs: globalParam, }; } - if (Array.isArray(this.sqlConfigurationParameters) && this.sqlConfigurationParameters.length) { + const sqlConfigurationParameters = utils.parseObject(this.sqlConfigurationParameters); + if (Array.isArray(sqlConfigurationParameters) && sqlConfigurationParameters.length) { payload.sql_configuration_parameters = { - configuration_pairs: this.sqlConfigurationParameters, + configuration_pairs: sqlConfigurationParameters, }; } - if (Array.isArray(this.dataAccessConfig) && this.dataAccessConfig.length) { - payload.data_access_config = this.dataAccessConfig; + const dataAccessConfig = utils.parseObject(this.dataAccessConfig); + if (Array.isArray(dataAccessConfig) && dataAccessConfig.length) { + payload.data_access_config = dataAccessConfig; } const response = await this.databricks.setSQLWarehouseConfig({ data: payload, diff --git a/components/databricks/common/constants.mjs b/components/databricks/common/constants.mjs new file mode 100644 index 0000000000000..4b53df7a27a86 --- /dev/null +++ b/components/databricks/common/constants.mjs @@ -0,0 +1,15 @@ +export const CLUSTER_SIZES = [ + "2X-Small", + "X-Small", + "Small", + "Medium", + "Large", + "X-Large", + "2X-Large", + "3X-Large", + "4X-Large", +]; + +export default { + CLUSTER_SIZES, +}; diff --git a/components/databricks/common/utils.mjs b/components/databricks/common/utils.mjs new file mode 100644 index 0000000000000..40bdddb456371 --- /dev/null +++ b/components/databricks/common/utils.mjs @@ -0,0 +1,31 @@ +const parseObject = (obj) => { + if (!obj) return {}; + + if (typeof obj === "string") { + try { + return JSON.parse(obj); + } catch (e) { + return obj; + } + } + + if (Array.isArray(obj)) { + return obj.map((item) => parseObject(item)); + } + + if (typeof obj === "object") { + return Object.fromEntries(Object.entries(obj).map(([ + key, + value, + ]) => [ + key, + parseObject(value), + ])); + } + + return obj; +}; + +export default { + parseObject, +}; From 2222816b7912a8453a69374a9e64436773053e11 Mon Sep 17 00:00:00 2001 From: Leo Vu Date: Tue, 2 Sep 2025 08:08:33 +0700 Subject: [PATCH 14/39] Add default value for maxNumClusters --- .../actions/create-sql-warehouse/create-sql-warehouse.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index d857beaeaec0d..bfdcc63bbc8e0 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -42,6 +42,7 @@ export default { label: "Max Number of Clusters", description: "Maximum number of clusters for autoscaler (≥ min_num_clusters and ≤ 30).", optional: true, + default: 1, }, enablePhoton: { type: "boolean", From 2aeacf22cf0419bfc01d0a464ea859c4979cdd1d Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Tue, 2 Sep 2025 12:17:02 +0530 Subject: [PATCH 15/39] create and edit sql warehouses fixes --- .../actions/create-sql-warehouse/create-sql-warehouse.mjs | 8 ++++++-- .../actions/edit-sql-warehouse/edit-sql-warehouse.mjs | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index bfdcc63bbc8e0..1ab590daec03f 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -133,13 +133,18 @@ export default { } const parsedTags = utils.parseObject(this.tags); - const tags = Object.entries(parsedTags).map(([ + const tagArray = Object.entries(parsedTags).map(([ key, value, ]) => ({ key, value, })); + if (tagArray.length) { + payload.tags = { + custom_tags: tagArray, + }; + } if (this.enablePhoton !== undefined) payload.enable_photon = this.enablePhoton; @@ -149,7 +154,6 @@ export default { if (this.spotInstancePolicy) payload.spot_instance_policy = this.spotInstancePolicy; if (this.channel) payload.channel = utils.parseObject(this.channel); - if (tags.length) payload.tags = tags; if (this.instanceProfileArn) payload.instance_profile_arn = this.instanceProfileArn; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 20d164dcc7fd7..f8586ea5c43e7 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -146,19 +146,20 @@ export default { } const parsedTags = utils.parseObject(this.tags); - const tags = Object.entries(parsedTags).map(([ + const tagArray = Object.entries(parsedTags).map(([ key, value, ]) => ({ key, value, })); - + if (tagArray.length) { + payload.tags = { custom_tags: tagArray }; + } if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy !== undefined) { payload.spot_instance_policy = this.spotInstancePolicy; } - if (tags.length) payload.tags = tags; if (this.channel !== undefined) payload.channel = utils.parseObject(this.channel); if (this.instanceProfileArn !== undefined) { payload.instance_profile_arn = this.instanceProfileArn; From 9bfe02300a1562e861c9c9ff2c6b004ec88bfa5e Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Tue, 2 Sep 2025 12:20:27 +0530 Subject: [PATCH 16/39] create and edit sql warehouse fixes --- .../actions/edit-sql-warehouse/edit-sql-warehouse.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index f8586ea5c43e7..a4e4cde5ef199 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -154,7 +154,9 @@ export default { value, })); if (tagArray.length) { - payload.tags = { custom_tags: tagArray }; + payload.tags = { + custom_tags: tagArray, + }; } if (this.warehouseType !== undefined) payload.warehouse_type = this.warehouseType; if (this.spotInstancePolicy !== undefined) { From 99dfc76c833e4b4d6b314183aa8eeb61b78707d8 Mon Sep 17 00:00:00 2001 From: Michelle Bergeron Date: Tue, 2 Sep 2025 11:41:22 -0400 Subject: [PATCH 17/39] updates --- .../create-sql-warehouse.mjs | 2 +- .../edit-sql-warehouse/edit-sql-warehouse.mjs | 6 ++---- .../set-sql-warehouse-config.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 16 ++++++---------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 1ab590daec03f..3efeb90ef1d60 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -83,7 +83,7 @@ export default { type: "object", label: "Channel", description: - "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CURRENT\", \"dbsql_version\": \"2023.35\" }`", + "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CUSTOM\", \"dbsql_version\": \"2023.35\" }`", optional: true, }, tags: { diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index a4e4cde5ef199..e61b6e987109c 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -92,7 +92,7 @@ export default { channel: { type: "object", label: "Channel", - description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CURRENT\", \"dbsql_version\": \"2023.35\" }`", + description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CUSTOM\", \"dbsql_version\": \"2023.35\" }`", optional: true, }, instanceProfileArn: { @@ -179,7 +179,5 @@ export default { $.export("$summary", `Successfully edited SQL Warehouse ID ${this.warehouseId}`); return response; - } - , - + }, }; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index b8e562a20d34d..730cd3b96cb26 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -36,7 +36,7 @@ export default { channel: { type: "object", label: "Channel", - description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_PREVIEW\", \"dbsql_version\": \"2023.35\" }`", + description: "Channel details. Example: `{ \"name\": \"CHANNEL_NAME_CUSTOM\", \"dbsql_version\": \"2023.35\" }`", optional: true, }, enabledWarehouseTypes: { diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 09707eb43e9e6..7fc5ae029df47 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -1,4 +1,6 @@ import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; export default { key: "databricks-set-sql-warehouse-permissions", @@ -22,17 +24,11 @@ export default { }, }, async run({ $ }) { - let acl = []; - try { - acl = (this.accessControlList || []) - .map((entry) => - typeof entry === "string" - ? JSON.parse(entry) - : entry) - .filter((entry) => entry && Object.keys(entry).length > 0); - } catch (err) { - throw new Error(`Invalid JSON in Access Control List: ${err.message}`); + let acl = utils.parseObject(this.accessControlList); + if (!Array.isArray(acl)) { + throw new ConfigurationError("Access Control List must be an array"); } + acl = acl.filter((entry) => entry && Object.keys(entry).length > 0); const response = await this.databricks.setSQLWarehousePermissions({ warehouseId: this.warehouseId, From 62287c709298478a7e00d2098270b96e7e44ee9d Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Thu, 4 Sep 2025 15:40:04 +0530 Subject: [PATCH 18/39] Added Vector Search Index API actions Summary This PR adds support for Databricks Vector Search Index APIs by implementing 9 new actions. Changes - Added create index action - Added get index action - Added list indexes action - Added delete index action - Added query index action - Added sync index action - Added scan index action - Added upsert index data action - Added delete index data action Resolves #18126 --- .../create-sql-warehouse.mjs | 2 +- .../create-vector-search-index.mjs | 120 ++++++++++++++++++ .../delete-sql-warehouse.mjs | 2 +- .../delete-vector-search-index-data.mjs | 46 +++++++ .../delete-vector-search-index.mjs | 27 ++++ .../edit-sql-warehouse/edit-sql-warehouse.mjs | 2 +- .../actions/get-run-output/get-run-output.mjs | 2 +- .../get-sql-warehouse-config.mjs | 2 +- .../get-sql-warehouse-permissions.mjs | 2 +- .../get-sql-warehouse/get-sql-warehouse.mjs | 2 +- .../get-vector-search-index.mjs | 31 +++++ .../actions/list-runs/list-runs.mjs | 2 +- .../list-sql-warehouses.mjs | 2 +- .../list-vector-search-indexes.mjs | 21 +++ .../query-vector-search-index.mjs | 91 +++++++++++++ .../actions/run-job-now/run-job-now.mjs | 2 +- .../scan-vector-search-index.mjs | 55 ++++++++ .../set-sql-warehouse-config.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 2 +- .../start-sql-warehouse.mjs | 2 +- .../stop-sql-warehouse/stop-sql-warehouse.mjs | 2 +- .../sync-vector-search-index.mjs | 31 +++++ .../upsert-vector-search-index-data.mjs | 55 ++++++++ components/databricks/databricks.app.mjs | 85 +++++++++++++ components/databricks/package.json | 2 +- 25 files changed, 577 insertions(+), 15 deletions(-) create mode 100644 components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs create mode 100644 components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs create mode 100644 components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs create mode 100644 components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs create mode 100644 components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs create mode 100644 components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs create mode 100644 components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs create mode 100644 components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs create mode 100644 components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 3efeb90ef1d60..b953d78090e61 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs new file mode 100644 index 0000000000000..78215f9482c54 --- /dev/null +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -0,0 +1,120 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; + +export default { + key: "databricks-create-vector-search-index", + name: "Create Vector Search Index", + description: + "Creates a new vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/createindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + name: { + type: "string", + label: "Index Name", + description: + "A unique name for the index (e.g., `main_catalog.docs.en_wiki_index`).", + }, + endpointName: { + type: "string", + label: "Endpoint Name", + description: "The vector search endpoint that will serve the index.", + }, + indexType: { + type: "string", + label: "Index Type", + description: "Type of index (`DELTA_SYNC` or `DIRECT_ACCESS`).", + options: ["DELTA_SYNC", "DIRECT_ACCESS"], + }, + primaryKey: { + type: "string", + label: "Primary Key", + description: "The primary key column for the index.", + }, + sourceTable: { + type: "string", + label: "Source Table", + description: + "The Delta table backing the index (required if `indexType` is `DELTA_SYNC`).", + optional: true, + }, + columnsToSync: { + type: "string[]", + label: "Columns to Sync", + description: + "List of columns to sync from the source Delta table. Example: `[\"id\", \"text\"]`", + optional: true, + }, + embeddingSourceColumns: { + type: "string[]", + label: "Embedding Source Columns", + description: + "List of embedding source column configs. Each entry should be a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`", + optional: true, + }, + pipelineType: { + type: "string", + label: "Pipeline Type", + description: "Pipeline type for syncing (default: TRIGGERED).", + options: ["TRIGGERED", "CONTINUOUS"], + optional: true, + default: "TRIGGERED", + }, + }, + + async run({ $ }) { + const payload = { + name: this.name, + endpoint_name: this.endpointName, + index_type: this.indexType, + primary_key: this.primaryKey, + }; + + if (this.indexType === "DELTA_SYNC") { + if (!this.sourceTable) { + throw new ConfigurationError( + "sourceTable is required when indexType is DELTA_SYNC." + ); + } + + const columnsToSync = utils.parseObject(this.columnsToSync); + const embeddingSourceColumns = utils.parseObject( + this.embeddingSourceColumns + ); + + if (!Array.isArray(columnsToSync) || !columnsToSync.length) { + throw new ConfigurationError( + "columnsToSync must be a non-empty array for DELTA_SYNC indexes." + ); + } + if ( + !Array.isArray(embeddingSourceColumns) || + !embeddingSourceColumns.length + ) { + throw new ConfigurationError( + "embeddingSourceColumns must be a non-empty array for DELTA_SYNC indexes." + ); + } + + payload.delta_sync_index_spec = { + source_table: this.sourceTable, + pipeline_type: this.pipelineType || "TRIGGERED", + columns_to_sync: columnsToSync, + embedding_source_columns: embeddingSourceColumns, + }; + } + + const response = await this.databricks.createVectorSearchIndex({ + data: payload, + $, + }); + + $.export( + "$summary", + `Successfully created vector search index: ${response?.name || this.name}` + ); + return response; + }, +}; diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 2eeea69a7713a..47584608434df 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-sql-warehouse", name: "Delete SQL Warehouse", description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs new file mode 100644 index 0000000000000..207466e2bbce7 --- /dev/null +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -0,0 +1,46 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-data-from-vector-index", + name: "Delete Data from Vector Search Index", + description: + "Deletes rows from a Direct Access vector index by primary-key values. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/deletedatavectorindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + primaryKeys: { + type: "string[]", + label: "Primary Keys", + description: + "Values of the index’s primary key column to delete (e.g. `1`, `2`). These are the values for the column you set as `primary_key` when the index was created.", + }, + }, + async run({ $ }) { + const keys = (this.primaryKeys || []) + .map((s) => String(s).trim()) + .filter(Boolean); + + if (!keys.length) { + throw new Error("Please provide at least one primary key to delete."); + } + + const response = await this.databricks.deleteVectorSearchData({ + indexName: this.indexName, + params: { primary_keys: keys }, + $, + }); + + $.export( + "$summary", + `Requested delete of ${keys.length} row(s) from index "${this.indexName}".`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs new file mode 100644 index 0000000000000..eee2834478e68 --- /dev/null +++ b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs @@ -0,0 +1,27 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-vector-search-index", + name: "Delete Vector Search Index", + description: "Deletes a vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/deleteindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + }, + async run({ $ }) { + const response = await this.databricks.deleteVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export("$summary", `Successfully deleted vector search index: ${this.indexName}`); + return response; + }, +}; diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index e61b6e987109c..293ee437f120a 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-run-output/get-run-output.mjs b/components/databricks/actions/get-run-output/get-run-output.mjs index ecfa86fcf0576..c44b1001a7bfa 100644 --- a/components/databricks/actions/get-run-output/get-run-output.mjs +++ b/components/databricks/actions/get-run-output/get-run-output.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run-output", name: "Get Run Output", description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)", - version: "0.0.2", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index 37202334854c5..1fbdda709e71b 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", description: "Retrieves the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 66ec10e7d75a7..4d76f5bb41c74 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", description: "Retrieves the permissions for a specific SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 3c5849846d41f..28d131c05c8f1 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse", name: "Get SQL Warehouse", description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/get)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs new file mode 100644 index 0000000000000..96623f3464444 --- /dev/null +++ b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs @@ -0,0 +1,31 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-vector-search-index", + name: "Get Vector Search Index", + description: "Retrieves details about a specific vector search index. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/getindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + }, + + async run({ $ }) { + const response = await this.databricks.getVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export( + "$summary", + `Successfully retrieved vector search index: ${this.indexName}`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/list-runs/list-runs.mjs b/components/databricks/actions/list-runs/list-runs.mjs index f24e52369b6e9..d2d7fe17cd3f3 100644 --- a/components/databricks/actions/list-runs/list-runs.mjs +++ b/components/databricks/actions/list-runs/list-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-runs", name: "List Runs", description: "Lists all runs available to the user. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.2", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index fbcf1865e22ed..356622572a9a9 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-sql-warehouses", name: "List SQL Warehouses", description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/list)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs new file mode 100644 index 0000000000000..1c93c9ad3c173 --- /dev/null +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -0,0 +1,21 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-list-vector-search-indexes", + name: "List Vector Search Indexes", + description: "Lists all vector search indexes in the workspace. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/listindexes)", + version: "0.0.1", + type: "action", + props: { + databricks, + }, + + async run({ $ }) { + const response = await this.databricks.listVectorSearchIndexes({ $ }); + + const count = response?.vector_indexes?.length || 0; + $.export("$summary", `Found ${count} vector search index${count === 1 ? "" : "es"}`); + + return response; + }, +}; diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs new file mode 100644 index 0000000000000..3925a793764bd --- /dev/null +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -0,0 +1,91 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-query-vector-search-index", + name: "Query Vector Search Index", + description: "Query a specific vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/queryindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + columns: { + type: "string[]", + label: "Columns", + description: "List of column names to include in the response.", + }, + queryText: { + type: "string", + label: "Query Text", + description: "Free-text query for semantic search.", + optional: true, + }, + queryVector: { + type: "string", + label: "Query Vector", + description: "JSON array of floats representing the embedding vector for the query.", + optional: true, + }, + filtersJson: { + type: "string", + label: "Filters JSON", + description: "JSON string representing query filters. Example: `{ \"id <\": 5 }`", + optional: true, + }, + numResults: { + type: "integer", + label: "Number of Results", + description: "Number of results to return. Defaults to 10.", + optional: true, + default: 10, + }, + includeEmbeddings: { + type: "boolean", + label: "Include Embeddings", + description: "Whether to include the embedding vectors in the results.", + optional: true, + }, + }, + + async run({ $ }) { + const payload = { + columns: this.columns, + num_results: this.numResults, + }; + + if (this.queryText) payload.query_text = this.queryText; + + if (this.queryVector) { + try { + payload.query_vector = JSON.parse(this.queryVector); + } catch (err) { + throw new Error(`Invalid queryVector JSON: ${err.message}`); + } + } + + if (this.filtersJson) { + payload.filters_json = utils.parseObject(this.filtersJson); + } + + if (this.includeEmbeddings !== undefined) { + payload.include_embeddings = this.includeEmbeddings; + } + + const response = await this.databricks.queryVectorSearchIndex({ + indexName: this.indexName, + data: payload, + $, + }); + + const count = response?.result?.data_array?.length || 0; + $.export("$summary", `Retrieved ${count} results from index ${this.indexName}`); + + return response; + }, +}; diff --git a/components/databricks/actions/run-job-now/run-job-now.mjs b/components/databricks/actions/run-job-now/run-job-now.mjs index 5de3e46e4ea59..4318cb98d9e8a 100644 --- a/components/databricks/actions/run-job-now/run-job-now.mjs +++ b/components/databricks/actions/run-job-now/run-job-now.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-run-job-now", name: "Run Job Now", description: "Run a job now and return the id of the triggered run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.2", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs new file mode 100644 index 0000000000000..5f257bb0df025 --- /dev/null +++ b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs @@ -0,0 +1,55 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-scan-vector-search-index", + name: "Scan Vector Search Index", + description: + "Scans a vector search index and returns entries after the given primary key. [See documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/scanindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + lastPrimaryKey: { + type: "string", + label: "Last Primary Key", + description: + "Primary key of the last entry returned in the previous scan. Leave empty to start from the beginning.", + optional: true, + }, + numResults: { + type: "integer", + label: "Number of Results", + description: "Number of results to return (defaults to 10).", + optional: true, + default: 10, + }, + }, + + async run({ $ }) { + const body = {}; + if (this.lastPrimaryKey !== undefined) { + body.last_primary_key = this.lastPrimaryKey; + } + if (this.numResults !== undefined) { + body.num_results = this.numResults; + } + + const response = await this.databricks.scanVectorSearchIndex({ + indexName: this.indexName, + data: body, + $, + }); + + $.export( + "$summary", + `Scanned index "${this.indexName}" and returned ${response?.data?.length ?? 0} entries.`, + ); + return response; + }, +}; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 730cd3b96cb26..6c9d4a7b98e6e 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", description: "Updates the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 7fc5ae029df47..202d1677dd5a9 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-permissions", name: "Set SQL Warehouse Permissions", description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/setpermissions)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 2de2b61cba5ac..669a593323040 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-start-sql-warehouse", name: "Start SQL Warehouse", description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/start)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index 1388ec5102b61..0117fca5d801d 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-stop-sql-warehouse", name: "Stop SQL Warehouse", description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/stop)", - version: "0.0.1", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs new file mode 100644 index 0000000000000..3c843af65a0fc --- /dev/null +++ b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs @@ -0,0 +1,31 @@ +import databricks from "../../databricks.app.mjs"; + +export default { + key: "databricks-sync-vector-search-index", + name: "Sync Vector Search Index", + description: "Synchronize a Delta Sync vector search index in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/syncindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + }, + async run({ $ }) { + const response = await this.databricks.syncVectorSearchIndex({ + indexName: this.indexName, + $, + }); + + $.export( + "$summary", + `Successfully triggered sync for vector search index: ${this.indexName}`, + ); + + return response; + }, +}; diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs new file mode 100644 index 0000000000000..cf36364d09659 --- /dev/null +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -0,0 +1,55 @@ +import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; +import { ConfigurationError } from "@pipedream/platform"; + +export default { + key: "databricks-upsert-vector-search-index", + name: "Upsert Vector Search Index Data", + description: "Upserts (inserts/updates) data into an existing vector search index. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/upsertdatavectorindex)", + version: "0.0.1", + type: "action", + props: { + databricks, + indexName: { + propDefinition: [ + databricks, + "indexName", + ], + }, + rows: { + type: "string[]", + label: "Rows to Upsert", + description: "Array of rows to upsert. Each row should be a JSON object string. Example: `{ \"id\": \"1\", \"text\": \"hello world\", \"text_vector\": [1.0, 2.0, 3.0] }`", + }, + }, + + async run({ $ }) { + const parsedRows = utils.parseObject(this.rows); + + if (!Array.isArray(parsedRows) || !parsedRows.length) { + throw new ConfigurationError("rows must be a non-empty array of JSON objects."); + } + + parsedRows.forEach((row, idx) => { + if (!row || typeof row !== "object") { + throw new ConfigurationError(`Row at index ${idx} is invalid. Each row must be a JSON object.`); + } + if (!row.id) { + throw new ConfigurationError(`Row at index ${idx} is missing required primary key field "id".`); + } + }); + + const payload = { + index_name: this.indexName, + inputs_json: JSON.stringify(parsedRows), + }; + + const response = await this.databricks.upsertVectorSearchIndex({ + data: payload, + $, + }); + + $.export("$summary", `Successfully upserted ${parsedRows.length} row(s) into index ${this.indexName}`); + return response; + }, +}; diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 05d6f88d697ab..9d4a377b96ddf 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -59,6 +59,20 @@ export default { })) || []; }, }, + indexName: { + type: "string", + label: "Vector Search Index", + description: "The name of the vector search index", + async options() { + const { vector_indexes } = await this.listVectorSearchIndexes(); + return vector_indexes?.map(({ + name: value, + }) => ({ + value, + label: value, + })) || []; + }, + }, }, methods: { _baseUrl() { @@ -199,5 +213,76 @@ export default { ...args, }); }, + createVectorSearchIndex(args = {}) { + return this._makeRequest({ + path: "/vector-search/indexes", + method: "POST", + ...args, + }); + }, + + getVectorSearchIndex({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}`, + method: "GET", + ...args, + }); + }, + + listVectorSearchIndexes(args = {}) { + return this._makeRequest({ + path: "/vector-search/indexes", + method: "GET", + ...args, + }); + }, + + deleteVectorSearchIndex({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}`, + method: "DELETE", + ...args, + }); + }, + + queryVectorSearchIndex({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/query`, + method: "POST", + ...args, + }); + }, + + syncVectorSearchIndex({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/sync`, + method: "POST", + ...args, + }); + }, + + deleteVectorSearchData({ indexName,...args }) { + return this._makeRequest({ + method: "DELETE", + url: `/api/2.0/vector-search/indexes/${indexName}/delete-data`, + ...args, + }); + }, + + upsertVectorSearchIndexData({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/upsert-data`, + method: "POST", + ...args, + }); + }, + + scanVectorSearchIndex({ indexName, ...args }) { + return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/scan`, + method: "POST", + ...args, + }); + }, }, }; diff --git a/components/databricks/package.json b/components/databricks/package.json index f49e7482191ef..e1cb32dbd8eaa 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.2.0", + "version": "0.3.0", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [ From ee33ab44c058d28dd85d0d7ab5b0dbc92b0fa505 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Thu, 4 Sep 2025 16:38:28 +0530 Subject: [PATCH 19/39] addressed coderabbit review feedback --- .../create-vector-search-index.mjs | 31 +++++++++----- .../delete-vector-search-index-data.mjs | 6 ++- .../get-vector-search-index.mjs | 2 +- .../list-vector-search-indexes.mjs | 8 +++- .../upsert-vector-search-index-data.mjs | 2 +- components/databricks/databricks.app.mjs | 40 ++++++++++++------- 6 files changed, 59 insertions(+), 30 deletions(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 78215f9482c54..1d85e6a8d2312 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -26,7 +26,10 @@ export default { type: "string", label: "Index Type", description: "Type of index (`DELTA_SYNC` or `DIRECT_ACCESS`).", - options: ["DELTA_SYNC", "DIRECT_ACCESS"], + options: [ + "DELTA_SYNC", + "DIRECT_ACCESS", + ], }, primaryKey: { type: "string", @@ -58,7 +61,10 @@ export default { type: "string", label: "Pipeline Type", description: "Pipeline type for syncing (default: TRIGGERED).", - options: ["TRIGGERED", "CONTINUOUS"], + options: [ + "TRIGGERED", + "CONTINUOUS", + ], optional: true, default: "TRIGGERED", }, @@ -75,18 +81,23 @@ export default { if (this.indexType === "DELTA_SYNC") { if (!this.sourceTable) { throw new ConfigurationError( - "sourceTable is required when indexType is DELTA_SYNC." + "sourceTable is required when indexType is DELTA_SYNC.", ); } + const columnsToSync = Array.isArray(this.columnsToSync) + ? this.columnsToSync + : utils.parseObject(this.columnsToSync); - const columnsToSync = utils.parseObject(this.columnsToSync); - const embeddingSourceColumns = utils.parseObject( - this.embeddingSourceColumns - ); + const embeddingSourceColumns = Array.isArray(this.embeddingSourceColumns) + ? this.embeddingSourceColumns.map((item) => + typeof item === "string" + ? JSON.parse(item) + : item) + : utils.parseObject(this.embeddingSourceColumns); if (!Array.isArray(columnsToSync) || !columnsToSync.length) { throw new ConfigurationError( - "columnsToSync must be a non-empty array for DELTA_SYNC indexes." + "columnsToSync must be a non-empty array for DELTA_SYNC indexes.", ); } if ( @@ -94,7 +105,7 @@ export default { !embeddingSourceColumns.length ) { throw new ConfigurationError( - "embeddingSourceColumns must be a non-empty array for DELTA_SYNC indexes." + "embeddingSourceColumns must be a non-empty array for DELTA_SYNC indexes.", ); } @@ -113,7 +124,7 @@ export default { $.export( "$summary", - `Successfully created vector search index: ${response?.name || this.name}` + `Successfully created vector search index: ${response?.name || this.name}`, ); return response; }, diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs index 207466e2bbce7..8c2510357de36 100644 --- a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -30,10 +30,12 @@ export default { if (!keys.length) { throw new Error("Please provide at least one primary key to delete."); } - + const response = await this.databricks.deleteVectorSearchData({ indexName: this.indexName, - params: { primary_keys: keys }, + data: { + primary_keys: keys, + }, $, }); diff --git a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs index 96623f3464444..6fa66762c1bbf 100644 --- a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs +++ b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs @@ -11,7 +11,7 @@ export default { indexName: { propDefinition: [ databricks, - "indexName", + "indexName", ], }, }, diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs index 1c93c9ad3c173..5f48fe8f13c2d 100644 --- a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -11,10 +11,14 @@ export default { }, async run({ $ }) { - const response = await this.databricks.listVectorSearchIndexes({ $ }); + const response = await this.databricks.listVectorSearchIndexes({ + $, + }); const count = response?.vector_indexes?.length || 0; - $.export("$summary", `Found ${count} vector search index${count === 1 ? "" : "es"}`); + $.export("$summary", `Found ${count} vector search index${count === 1 + ? "" + : "es"}`); return response; }, diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs index cf36364d09659..3f55184300d54 100644 --- a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -13,7 +13,7 @@ export default { indexName: { propDefinition: [ databricks, - "indexName", + "indexName", ], }, rows: { diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 9d4a377b96ddf..b1f624397ea21 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -59,17 +59,15 @@ export default { })) || []; }, }, - indexName: { + indexName: { type: "string", label: "Vector Search Index", description: "The name of the vector search index", async options() { const { vector_indexes } = await this.listVectorSearchIndexes(); - return vector_indexes?.map(({ - name: value, - }) => ({ + return vector_indexes?.map(({ name: value }) => ({ value, - label: value, + label: value, })) || []; }, }, @@ -221,7 +219,9 @@ export default { }); }, - getVectorSearchIndex({ indexName, ...args }) { + getVectorSearchIndex({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}`, method: "GET", @@ -237,7 +237,9 @@ export default { }); }, - deleteVectorSearchIndex({ indexName, ...args }) { + deleteVectorSearchIndex({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}`, method: "DELETE", @@ -245,7 +247,9 @@ export default { }); }, - queryVectorSearchIndex({ indexName, ...args }) { + queryVectorSearchIndex({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}/query`, method: "POST", @@ -253,7 +257,9 @@ export default { }); }, - syncVectorSearchIndex({ indexName, ...args }) { + syncVectorSearchIndex({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}/sync`, method: "POST", @@ -261,15 +267,19 @@ export default { }); }, - deleteVectorSearchData({ indexName,...args }) { + deleteVectorSearchData({ + indexName, ...args + }) { return this._makeRequest({ + path: `/vector-search/indexes/${indexName}/delete-data`, method: "DELETE", - url: `/api/2.0/vector-search/indexes/${indexName}/delete-data`, - ...args, + ...args, }); }, - upsertVectorSearchIndexData({ indexName, ...args }) { + upsertVectorSearchIndexData({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}/upsert-data`, method: "POST", @@ -277,7 +287,9 @@ export default { }); }, - scanVectorSearchIndex({ indexName, ...args }) { + scanVectorSearchIndex({ + indexName, ...args + }) { return this._makeRequest({ path: `/vector-search/indexes/${indexName}/scan`, method: "POST", From a13c04cf9e8142eabbc7855d9ce3d5eda8eacb82 Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 8 Sep 2025 16:37:46 +0530 Subject: [PATCH 20/39] version updated --- .../databricks/actions/create-endpoint/create-endpoint.mjs | 2 +- .../actions/create-sql-warehouse/create-sql-warehouse.mjs | 2 +- .../databricks/actions/delete-endpoint/delete-endpoint.mjs | 2 +- components/databricks/actions/get-endpoint/get-endpoint.mjs | 2 +- components/databricks/actions/list-endpoints/list-endpoints.mjs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/databricks/actions/create-endpoint/create-endpoint.mjs b/components/databricks/actions/create-endpoint/create-endpoint.mjs index 3777a7ca32c5e..33bf7ad4d7a8f 100644 --- a/components/databricks/actions/create-endpoint/create-endpoint.mjs +++ b/components/databricks/actions/create-endpoint/create-endpoint.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-endpoint", name: "Create Endpoint", description: "Create a new vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/createendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index 01176403df86c..b953d78090e61 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", - version: "0.0.3" + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs index 5ad2a8c43304f..4a95bb6f13e2d 100644 --- a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs +++ b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-endpoint", name: "Delete Endpoint", description: "Delete a vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/deleteendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-endpoint/get-endpoint.mjs b/components/databricks/actions/get-endpoint/get-endpoint.mjs index 99fd6b71abb96..1f494f4658be1 100644 --- a/components/databricks/actions/get-endpoint/get-endpoint.mjs +++ b/components/databricks/actions/get-endpoint/get-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-endpoint", name: "Get Endpoint", description: "Get details of a specific vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/getendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-endpoints/list-endpoints.mjs b/components/databricks/actions/list-endpoints/list-endpoints.mjs index ed06bdebf9710..c0fe92e1672e6 100644 --- a/components/databricks/actions/list-endpoints/list-endpoints.mjs +++ b/components/databricks/actions/list-endpoints/list-endpoints.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-endpoints", name: "List Endpoints", description: "List all vector search endpoints. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/listendpoints)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, From 30e5b6c32e91f7f0fd867b9a01e6db7b05a8a9cb Mon Sep 17 00:00:00 2001 From: Lokesh Chand Date: Mon, 8 Sep 2025 16:56:29 +0530 Subject: [PATCH 21/39] resolved the linting issues --- .../delete-vector-search-index-data.mjs | 2 +- .../upsert-vector-search-index-data.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs index 8c2510357de36..3a33ee8be9075 100644 --- a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -1,7 +1,7 @@ import databricks from "../../databricks.app.mjs"; export default { - key: "databricks-delete-data-from-vector-index", + key: "databricks-delete-vector-search-index-data", name: "Delete Data from Vector Search Index", description: "Deletes rows from a Direct Access vector index by primary-key values. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/deletedatavectorindex)", diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs index 3f55184300d54..eb2c995eb7003 100644 --- a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -3,7 +3,7 @@ import utils from "../../common/utils.mjs"; import { ConfigurationError } from "@pipedream/platform"; export default { - key: "databricks-upsert-vector-search-index", + key: "databricks-upsert-vector-search-index-data", name: "Upsert Vector Search Index Data", description: "Upserts (inserts/updates) data into an existing vector search index. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/upsertdatavectorindex)", version: "0.0.1", From 47fe441f0bfde51ee7f4ac366c9da67bf2ac7e87 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Sat, 13 Sep 2025 23:07:09 +0530 Subject: [PATCH 22/39] addressed all test failures --- .../create-vector-search-index.mjs | 37 ++++++++++----- .../list-vector-search-indexes.mjs | 45 +++++++++++++++---- .../query-vector-search-index.mjs | 2 +- 3 files changed, 65 insertions(+), 19 deletions(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 1d85e6a8d2312..6133aff800f7f 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -18,9 +18,10 @@ export default { "A unique name for the index (e.g., `main_catalog.docs.en_wiki_index`).", }, endpointName: { - type: "string", - label: "Endpoint Name", - description: "The vector search endpoint that will serve the index.", + propDefinition: [ + databricks, + "endpointName", + ], }, indexType: { type: "string", @@ -40,21 +41,28 @@ export default { type: "string", label: "Source Table", description: - "The Delta table backing the index (required if `indexType` is `DELTA_SYNC`).", + "The Delta table backing the index (required for `DELTA_SYNC`).", optional: true, }, columnsToSync: { type: "string[]", label: "Columns to Sync", description: - "List of columns to sync from the source Delta table. Example: `[\"id\", \"text\"]`", + "List of columns to sync from the source Delta table. Example: `[\"id\", \"text\"]` (required for `DELTA_SYNC`).", optional: true, }, embeddingSourceColumns: { type: "string[]", label: "Embedding Source Columns", description: - "List of embedding source column configs. Each entry should be a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`", + "List of embedding source column configs. Each entry should be a JSON object string like `[ { \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" } ]` (required for `DELTA_SYNC`).", + optional: true, + }, + schemaJson: { + type: "string", + label: "Schema JSON", + description: + "The schema of the index in JSON format. Example: `{ \"columns\": [{ \"name\": \"id\", \"type\": \"string\" }, { \"name\": \"text_vector\", \"type\": \"array\" }] }`. Required for `DIRECT_ACCESS` indexes.", optional: true, }, pipelineType: { @@ -84,6 +92,7 @@ export default { "sourceTable is required when indexType is DELTA_SYNC.", ); } + const columnsToSync = Array.isArray(this.columnsToSync) ? this.columnsToSync : utils.parseObject(this.columnsToSync); @@ -100,10 +109,7 @@ export default { "columnsToSync must be a non-empty array for DELTA_SYNC indexes.", ); } - if ( - !Array.isArray(embeddingSourceColumns) || - !embeddingSourceColumns.length - ) { + if (!Array.isArray(embeddingSourceColumns) || !embeddingSourceColumns.length) { throw new ConfigurationError( "embeddingSourceColumns must be a non-empty array for DELTA_SYNC indexes.", ); @@ -117,6 +123,17 @@ export default { }; } + else if (this.indexType === "DIRECT_ACCESS") { + if (!this.schemaJson) { + throw new ConfigurationError( + "schemaJson is required when indexType is DIRECT_ACCESS.", + ); + } + payload.direct_access_index_spec = { + schema_json: this.schemaJson, + }; + } + const response = await this.databricks.createVectorSearchIndex({ data: payload, $, diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs index 5f48fe8f13c2d..99446353cb35d 100644 --- a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -8,18 +8,47 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, }, async run({ $ }) { - const response = await this.databricks.listVectorSearchIndexes({ - $, - }); + const allIndexes = []; + let pageToken; - const count = response?.vector_indexes?.length || 0; - $.export("$summary", `Found ${count} vector search index${count === 1 - ? "" - : "es"}`); + do { + const { + vector_indexes, next_page_token, + } = await this.databricks.listIndexes({ + params: { + endpoint_name: this.endpointName, + ...(pageToken + ? { + page_token: pageToken, + } + : {}), + }, + $, + }); - return response; + if (vector_indexes?.length) { + allIndexes.push(...vector_indexes); + } + + pageToken = next_page_token; + } while (pageToken); + + $.export( + "$summary", + `Successfully retrieved ${allIndexes.length} index${allIndexes.length === 1 + ? "" + : "es"}.`, + ); + + return allIndexes; }, }; diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs index 3925a793764bd..fb957b2d8bf7c 100644 --- a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -18,7 +18,7 @@ export default { columns: { type: "string[]", label: "Columns", - description: "List of column names to include in the response.", + description: "List of column names to include in the response. Example: `[\"id\"]`", }, queryText: { type: "string", From 9bcacc35ed762c2303a3e4cb9d29e69e319b436d Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Sat, 13 Sep 2025 23:32:45 +0530 Subject: [PATCH 23/39] addressed coderabbit review feedback --- .../create-vector-search-index.mjs | 16 ++++++++++++---- .../list-vector-search-indexes.mjs | 2 +- .../query-vector-search-index.mjs | 13 ++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 6133aff800f7f..7e6f37355df72 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -98,10 +98,18 @@ export default { : utils.parseObject(this.columnsToSync); const embeddingSourceColumns = Array.isArray(this.embeddingSourceColumns) - ? this.embeddingSourceColumns.map((item) => - typeof item === "string" - ? JSON.parse(item) - : item) + ? this.embeddingSourceColumns.map((item, idx) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + throw new ConfigurationError( + `embeddingSourceColumns[${idx}] is not valid JSON: ${e.message}`, + ); + } + } + return item; + }) : utils.parseObject(this.embeddingSourceColumns); if (!Array.isArray(columnsToSync) || !columnsToSync.length) { diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs index 99446353cb35d..33163e281fc12 100644 --- a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -23,7 +23,7 @@ export default { do { const { vector_indexes, next_page_token, - } = await this.databricks.listIndexes({ + } = await this.databricks.listVectorSearchIndexes({ params: { endpoint_name: this.endpointName, ...(pageToken diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs index fb957b2d8bf7c..3f224164980ad 100644 --- a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -64,13 +64,24 @@ export default { if (this.queryVector) { try { payload.query_vector = JSON.parse(this.queryVector); + if ( + !Array.isArray(payload.query_vector) || + payload.query_vector.length === 0 || + !payload.query_vector.every((n) => typeof n === "number" && Number.isFinite(n)) + ) { + throw new Error("`queryVector` must be a non-empty JSON array of finite numbers."); + } } catch (err) { throw new Error(`Invalid queryVector JSON: ${err.message}`); } } if (this.filtersJson) { - payload.filters_json = utils.parseObject(this.filtersJson); + try { + payload.filters_json = utils.parseObject(this.filtersJson); + } catch (err) { + throw new Error(`Invalid filtersJson: ${err.message}`); + } } if (this.includeEmbeddings !== undefined) { From 13a1c59c46512821a9bf391c6d4955c44fa9831c Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Sat, 13 Sep 2025 23:57:02 +0530 Subject: [PATCH 24/39] addressed coderabbit review feedback --- .../create-vector-search-index.mjs | 64 ++++++++++++++++--- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 7e6f37355df72..eed020bc8b228 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -97,6 +97,7 @@ export default { ? this.columnsToSync : utils.parseObject(this.columnsToSync); + // Parse embeddingSourceColumns (either object[] or string[] of JSON) const embeddingSourceColumns = Array.isArray(this.embeddingSourceColumns) ? this.embeddingSourceColumns.map((item, idx) => { if (typeof item === "string") { @@ -112,23 +113,66 @@ export default { }) : utils.parseObject(this.embeddingSourceColumns); - if (!Array.isArray(columnsToSync) || !columnsToSync.length) { - throw new ConfigurationError( - "columnsToSync must be a non-empty array for DELTA_SYNC indexes.", - ); - } - if (!Array.isArray(embeddingSourceColumns) || !embeddingSourceColumns.length) { + // Parse embeddingVectorColumns (either object[] or string[] of JSON) + const embeddingVectorColumns = Array.isArray(this.embeddingVectorColumns) + ? this.embeddingVectorColumns.map((item, idx) => { + if (typeof item === "string") { + try { + return JSON.parse(item); + } catch (e) { + throw new ConfigurationError( + `embeddingVectorColumns[${idx}] is not valid JSON: ${e.message}`, + ); + } + } + return item; + }) + : utils.parseObject(this.embeddingVectorColumns); + + // Require at least one embedding config: source OR vector columns + const hasSource = Array.isArray(embeddingSourceColumns) && embeddingSourceColumns.length > 0; + const hasVectors = Array.isArray(embeddingVectorColumns) && embeddingVectorColumns.length > 0; + if (!hasSource && !hasVectors) { throw new ConfigurationError( - "embeddingSourceColumns must be a non-empty array for DELTA_SYNC indexes.", + "Provide either embeddingSourceColumns (compute embeddings) or embeddingVectorColumns (self-managed) for DELTA_SYNC indexes.", ); } - payload.delta_sync_index_spec = { + const deltaSpec = { source_table: this.sourceTable, pipeline_type: this.pipelineType || "TRIGGERED", - columns_to_sync: columnsToSync, - embedding_source_columns: embeddingSourceColumns, }; + if (Array.isArray(columnsToSync) && columnsToSync.length > 0) { + deltaSpec.columns_to_sync = columnsToSync; + } + if (hasSource) { + // Optional: shallow validation of required keys + for (const [ + i, + c, + ] of embeddingSourceColumns.entries()) { + if (!c?.name || !c?.embedding_model_endpoint_name) { + throw new ConfigurationError( + `embeddingSourceColumns[${i}] must include "name" and "embedding_model_endpoint_name"`, + ); + } + } + deltaSpec.embedding_source_columns = embeddingSourceColumns; + } + if (hasVectors) { + for (const [ + i, + c, + ] of embeddingVectorColumns.entries()) { + if (!c?.name || typeof c?.embedding_dimension !== "number") { + throw new ConfigurationError( + `embeddingVectorColumns[${i}] must include "name" and numeric "embedding_dimension"`, + ); + } + } + deltaSpec.embedding_vector_columns = embeddingVectorColumns; + } + payload.delta_sync_index_spec = deltaSpec; } else if (this.indexType === "DIRECT_ACCESS") { From 9ea188ad666ea2f9b261d639fa86eb3b7942fb7e Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Sun, 14 Sep 2025 21:41:14 +0530 Subject: [PATCH 25/39] addressed coderabbit review feedback --- .../create-vector-search-index.mjs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index eed020bc8b228..8f523ec39ced8 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -55,7 +55,14 @@ export default { type: "string[]", label: "Embedding Source Columns", description: - "List of embedding source column configs. Each entry should be a JSON object string like `[ { \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" } ]` (required for `DELTA_SYNC`).", + "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`. Provide when Databricks computes embeddings (DELTA_SYNC).", + optional: true, + }, + embeddingVectorColumns: { + type: "string[]", + label: "Embedding Vector Columns", + description: + "List of self-managed vector column configs. Each entry is a JSON object string like `{ \"name\": \"text_vector\", \"embedding_dimension\": 1536 }`. Provide when you manage embeddings yourself (DELTA_SYNC).", optional: true, }, schemaJson: { @@ -137,6 +144,11 @@ export default { "Provide either embeddingSourceColumns (compute embeddings) or embeddingVectorColumns (self-managed) for DELTA_SYNC indexes.", ); } + if (hasSource && hasVectors) { + throw new ConfigurationError( + "Provide only one of embeddingSourceColumns or embeddingVectorColumns for DELTA_SYNC indexes.", + ); + } const deltaSpec = { source_table: this.sourceTable, From ebea51005fa1d883dfdaf83b26f5ffa44de36c5f Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Mon, 15 Sep 2025 11:36:28 +0530 Subject: [PATCH 26/39] updated --- .../create-vector-search-index.mjs | 65 +------------------ 1 file changed, 3 insertions(+), 62 deletions(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 8f523ec39ced8..2dd8cb51b96e6 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -58,13 +58,6 @@ export default { "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`. Provide when Databricks computes embeddings (DELTA_SYNC).", optional: true, }, - embeddingVectorColumns: { - type: "string[]", - label: "Embedding Vector Columns", - description: - "List of self-managed vector column configs. Each entry is a JSON object string like `{ \"name\": \"text_vector\", \"embedding_dimension\": 1536 }`. Provide when you manage embeddings yourself (DELTA_SYNC).", - optional: true, - }, schemaJson: { type: "string", label: "Schema JSON", @@ -104,49 +97,11 @@ export default { ? this.columnsToSync : utils.parseObject(this.columnsToSync); - // Parse embeddingSourceColumns (either object[] or string[] of JSON) - const embeddingSourceColumns = Array.isArray(this.embeddingSourceColumns) - ? this.embeddingSourceColumns.map((item, idx) => { - if (typeof item === "string") { - try { - return JSON.parse(item); - } catch (e) { - throw new ConfigurationError( - `embeddingSourceColumns[${idx}] is not valid JSON: ${e.message}`, - ); - } - } - return item; - }) - : utils.parseObject(this.embeddingSourceColumns); - - // Parse embeddingVectorColumns (either object[] or string[] of JSON) - const embeddingVectorColumns = Array.isArray(this.embeddingVectorColumns) - ? this.embeddingVectorColumns.map((item, idx) => { - if (typeof item === "string") { - try { - return JSON.parse(item); - } catch (e) { - throw new ConfigurationError( - `embeddingVectorColumns[${idx}] is not valid JSON: ${e.message}`, - ); - } - } - return item; - }) - : utils.parseObject(this.embeddingVectorColumns); - - // Require at least one embedding config: source OR vector columns + const embeddingSourceColumns = utils.parseObject(this.embeddingSourceColumns); const hasSource = Array.isArray(embeddingSourceColumns) && embeddingSourceColumns.length > 0; - const hasVectors = Array.isArray(embeddingVectorColumns) && embeddingVectorColumns.length > 0; - if (!hasSource && !hasVectors) { + if (!hasSource) { throw new ConfigurationError( - "Provide either embeddingSourceColumns (compute embeddings) or embeddingVectorColumns (self-managed) for DELTA_SYNC indexes.", - ); - } - if (hasSource && hasVectors) { - throw new ConfigurationError( - "Provide only one of embeddingSourceColumns or embeddingVectorColumns for DELTA_SYNC indexes.", + "embeddingSourceColumns is required when indexType is DELTA_SYNC.", ); } @@ -158,7 +113,6 @@ export default { deltaSpec.columns_to_sync = columnsToSync; } if (hasSource) { - // Optional: shallow validation of required keys for (const [ i, c, @@ -171,19 +125,6 @@ export default { } deltaSpec.embedding_source_columns = embeddingSourceColumns; } - if (hasVectors) { - for (const [ - i, - c, - ] of embeddingVectorColumns.entries()) { - if (!c?.name || typeof c?.embedding_dimension !== "number") { - throw new ConfigurationError( - `embeddingVectorColumns[${i}] must include "name" and numeric "embedding_dimension"`, - ); - } - } - deltaSpec.embedding_vector_columns = embeddingVectorColumns; - } payload.delta_sync_index_spec = deltaSpec; } From 75214fc1122ed74b3f99e8c4fcef7aad4ef9a707 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Mon, 15 Sep 2025 17:26:25 +0530 Subject: [PATCH 27/39] updated --- .../delete-vector-search-index-data.mjs | 11 ++++-- .../list-vector-search-indexes.mjs | 36 +++++-------------- components/databricks/databricks.app.mjs | 13 ++++--- 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs index 3a33ee8be9075..1c1b852dd11e9 100644 --- a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -1,4 +1,5 @@ import databricks from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; export default { key: "databricks-delete-vector-search-index-data", @@ -23,7 +24,13 @@ export default { }, }, async run({ $ }) { - const keys = (this.primaryKeys || []) + const parsedKeys = utils.parseObject(this.primaryKeys); + + const keys = (Array.isArray(parsedKeys) + ? parsedKeys + : [ + parsedKeys, + ]) .map((s) => String(s).trim()) .filter(Boolean); @@ -33,7 +40,7 @@ export default { const response = await this.databricks.deleteVectorSearchData({ indexName: this.indexName, - data: { + params: { primary_keys: keys, }, $, diff --git a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs index 33163e281fc12..28e71eb08d0a1 100644 --- a/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs +++ b/components/databricks/actions/list-vector-search-indexes/list-vector-search-indexes.mjs @@ -3,7 +3,7 @@ import databricks from "../../databricks.app.mjs"; export default { key: "databricks-list-vector-search-indexes", name: "List Vector Search Indexes", - description: "Lists all vector search indexes in the workspace. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/listindexes)", + description: "Lists all vector search indexes for a given endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchindexes/listindexes)", version: "0.0.1", type: "action", props: { @@ -17,38 +17,20 @@ export default { }, async run({ $ }) { - const allIndexes = []; - let pageToken; - - do { - const { - vector_indexes, next_page_token, - } = await this.databricks.listVectorSearchIndexes({ - params: { - endpoint_name: this.endpointName, - ...(pageToken - ? { - page_token: pageToken, - } - : {}), - }, - $, - }); - - if (vector_indexes?.length) { - allIndexes.push(...vector_indexes); - } - - pageToken = next_page_token; - } while (pageToken); + const { vector_indexes = [] } = await this.databricks.listVectorSearchIndexes({ + params: { + endpoint_name: this.endpointName, + }, + $, + }); $.export( "$summary", - `Successfully retrieved ${allIndexes.length} index${allIndexes.length === 1 + `Successfully retrieved ${vector_indexes.length} index${vector_indexes.length === 1 ? "" : "es"}.`, ); - return allIndexes; + return vector_indexes; }, }; diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 1576ba76b5841..b16eaec6ab355 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -281,9 +281,11 @@ export default { }); }, - listVectorSearchIndexes(args = {}) { + listVectorSearchIndexes({ + params, ...args + }) { return this._makeRequest({ - path: "/vector-search/indexes", + path: `/vector-search/indexes?endpoint_name=${params.endpoint_name}`, method: "GET", ...args, }); @@ -320,10 +322,11 @@ export default { }, deleteVectorSearchData({ - indexName, ...args - }) { + indexName, params, ...args + }) + { return this._makeRequest({ - path: `/vector-search/indexes/${indexName}/delete-data`, + path: `/vector-search/indexes/${indexName}/delete-data?primary_keys=${params.primary_keys}`, method: "DELETE", ...args, }); From 866676467dd82f80792b00e960cddf517e22db18 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Mon, 15 Sep 2025 20:42:25 +0530 Subject: [PATCH 28/39] updates --- components/databricks/databricks.app.mjs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index b16eaec6ab355..3f8bfd6408a26 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -84,12 +84,20 @@ export default { type: "string", label: "Vector Search Index", description: "The name of the vector search index", - async options() { - const { vector_indexes } = await this.listVectorSearchIndexes(); - return vector_indexes?.map(({ name: value }) => ({ - value, - label: value, - })) || []; + async options({ props }) { + if (!props.endpointName) { + return []; + } + const { vector_indexes = [] } = await this.listVectorSearchIndexes({ + params: { + endpoint_name: props.endpointName, + }, + }); + + return vector_indexes.map(({ name }) => ({ + value: name, + label: name, + })); }, }, }, From 30b514d6c72c427dcc1d43d06d6e3cf6ffe8c085 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Tue, 16 Sep 2025 18:09:31 +0530 Subject: [PATCH 29/39] fixed failed test cases --- components/databricks/databricks.app.mjs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 3f8bfd6408a26..e5eee56c217c6 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -50,7 +50,7 @@ export default { description: "The name of the vector search endpoint", async options({ prevContext }) { const { - endpoints, next_page_token, + endpoints = [], next_page_token, } = await this.listEndpoints({ params: { page_token: prevContext.page_token, @@ -82,15 +82,12 @@ export default { }, indexName: { type: "string", - label: "Vector Search Index", + label: "Index Name", description: "The name of the vector search index", - async options({ props }) { - if (!props.endpointName) { - return []; - } + async options({ endpointName }) { const { vector_indexes = [] } = await this.listVectorSearchIndexes({ params: { - endpoint_name: props.endpointName, + endpoint_name: endpointName, }, }); @@ -293,8 +290,9 @@ export default { params, ...args }) { return this._makeRequest({ - path: `/vector-search/indexes?endpoint_name=${params.endpoint_name}`, + path: "/vector-search/indexes", method: "GET", + params, ...args, }); }, @@ -334,8 +332,9 @@ export default { }) { return this._makeRequest({ - path: `/vector-search/indexes/${indexName}/delete-data?primary_keys=${params.primary_keys}`, + path: `/vector-search/indexes/${indexName}/delete-data`, method: "DELETE", + params, ...args, }); }, From 86b4d47fe9bb665c6710454a6a878173890e441b Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Wed, 17 Sep 2025 18:10:34 +0530 Subject: [PATCH 30/39] updated --- .../create-vector-search-index.mjs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 2dd8cb51b96e6..e2b6ed9b4fa3b 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -55,7 +55,7 @@ export default { type: "string[]", label: "Embedding Source Columns", description: - "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`. Provide when Databricks computes embeddings (DELTA_SYNC).", + "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`.", optional: true, }, schemaJson: { @@ -87,6 +87,11 @@ export default { }; if (this.indexType === "DELTA_SYNC") { + if (this.schemaJson) { + throw new ConfigurationError( + "`Schema JSON` is not allowed when indexType is DELTA_SYNC.", + ); + } if (!this.sourceTable) { throw new ConfigurationError( "sourceTable is required when indexType is DELTA_SYNC.", @@ -129,6 +134,11 @@ export default { } else if (this.indexType === "DIRECT_ACCESS") { + if (this.sourceTable || this.columnsToSync?.length) { + throw new ConfigurationError( + "`Source Table`,`Embedding Source Columns` and `Columns to Sync` are not allowed when indexType is DIRECT_ACCESS.", + ); + } if (!this.schemaJson) { throw new ConfigurationError( "schemaJson is required when indexType is DIRECT_ACCESS.", From da2fa277639d7d6f0e635ffb69708dad3e317ab9 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Wed, 17 Sep 2025 18:13:49 +0530 Subject: [PATCH 31/39] updated --- .../create-vector-search-index/create-vector-search-index.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index e2b6ed9b4fa3b..146409213b419 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -55,7 +55,7 @@ export default { type: "string[]", label: "Embedding Source Columns", description: - "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`.", + "List of embedding source column configs. Each entry is a JSON object string like `{ \"embedding_model_endpoint_name\": \"e5-small-v2\", \"name\": \"text\" }`.Provide when Databricks computes embeddings (DELTA_SYNC).", optional: true, }, schemaJson: { From f9aa39ef831e30a58a6d8abb50bd6a060d8b1693 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Wed, 17 Sep 2025 18:27:53 +0530 Subject: [PATCH 32/39] updated --- .../create-vector-search-index/create-vector-search-index.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs index 146409213b419..492bbca32e0dd 100644 --- a/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs +++ b/components/databricks/actions/create-vector-search-index/create-vector-search-index.mjs @@ -134,7 +134,7 @@ export default { } else if (this.indexType === "DIRECT_ACCESS") { - if (this.sourceTable || this.columnsToSync?.length) { + if (this.sourceTable || this.columnsToSync?.length || this.embeddingSourceColumns?.length) { throw new ConfigurationError( "`Source Table`,`Embedding Source Columns` and `Columns to Sync` are not allowed when indexType is DIRECT_ACCESS.", ); From 008e23b38dd7043be1b65d74d97b035c2849d4f8 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Wed, 17 Sep 2025 21:49:04 +0530 Subject: [PATCH 33/39] fixed failed test cases --- .../delete-vector-search-index-data.mjs | 9 +++++++++ .../delete-vector-search-index.mjs | 9 +++++++++ .../get-vector-search-index/get-vector-search-index.mjs | 9 +++++++++ .../query-vector-search-index.mjs | 9 +++++++++ .../scan-vector-search-index.mjs | 9 +++++++++ .../sync-vector-search-index.mjs | 9 +++++++++ .../upsert-vector-search-index-data.mjs | 9 +++++++++ components/databricks/databricks.app.mjs | 3 +++ 8 files changed, 66 insertions(+) diff --git a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs index 1c1b852dd11e9..b605f15edb7c3 100644 --- a/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs +++ b/components/databricks/actions/delete-vector-search-index-data/delete-vector-search-index-data.mjs @@ -10,10 +10,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, primaryKeys: { diff --git a/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs index eee2834478e68..756f529d003b0 100644 --- a/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs +++ b/components/databricks/actions/delete-vector-search-index/delete-vector-search-index.mjs @@ -8,10 +8,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, }, diff --git a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs index 6fa66762c1bbf..6ead8475541e1 100644 --- a/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs +++ b/components/databricks/actions/get-vector-search-index/get-vector-search-index.mjs @@ -8,10 +8,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, }, diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs index 3f224164980ad..117d2adce6dcb 100644 --- a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -9,10 +9,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, columns: { diff --git a/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs index 5f257bb0df025..c1036d0d10220 100644 --- a/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs +++ b/components/databricks/actions/scan-vector-search-index/scan-vector-search-index.mjs @@ -9,10 +9,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, lastPrimaryKey: { diff --git a/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs index 3c843af65a0fc..ceb744671c506 100644 --- a/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs +++ b/components/databricks/actions/sync-vector-search-index/sync-vector-search-index.mjs @@ -8,10 +8,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, }, diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs index eb2c995eb7003..7ca763be4ecab 100644 --- a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -10,10 +10,19 @@ export default { type: "action", props: { databricks, + endpointName: { + propDefinition: [ + databricks, + "endpointName", + ], + }, indexName: { propDefinition: [ databricks, "indexName", + ({ endpointName }) => ({ + endpointName, + }), ], }, rows: { diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index e5eee56c217c6..c2f3a65bcb915 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -85,6 +85,9 @@ export default { label: "Index Name", description: "The name of the vector search index", async options({ endpointName }) { + if (!endpointName) { + return []; + } const { vector_indexes = [] } = await this.listVectorSearchIndexes({ params: { endpoint_name: endpointName, From 8fb30727a060afeb8d42b2b353ccdaa97b371583 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Thu, 18 Sep 2025 17:46:19 +0530 Subject: [PATCH 34/39] fixed failed test cases --- .../query-vector-search-index.mjs | 7 +------ .../upsert-vector-search-index-data.mjs | 2 +- components/databricks/databricks.app.mjs | 3 +++ 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs index 117d2adce6dcb..44f8cfac7685d 100644 --- a/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs +++ b/components/databricks/actions/query-vector-search-index/query-vector-search-index.mjs @@ -1,5 +1,4 @@ import databricks from "../../databricks.app.mjs"; -import utils from "../../common/utils.mjs"; export default { key: "databricks-query-vector-search-index", @@ -86,11 +85,7 @@ export default { } if (this.filtersJson) { - try { - payload.filters_json = utils.parseObject(this.filtersJson); - } catch (err) { - throw new Error(`Invalid filtersJson: ${err.message}`); - } + payload.filters_json = this.filtersJson; } if (this.includeEmbeddings !== undefined) { diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs index 7ca763be4ecab..c3134d9f1a8ba 100644 --- a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -53,7 +53,7 @@ export default { inputs_json: JSON.stringify(parsedRows), }; - const response = await this.databricks.upsertVectorSearchIndex({ + const response = await this.databricks.upsertVectorSearchIndexData({ data: payload, $, }); diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index c2f3a65bcb915..5f12cc346976b 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -119,6 +119,9 @@ export default { return axios($, { url: `${this._baseUrl()}${path}`, headers: this._headers(), + paramsSerializer: { + indexes: null, + }, ...args, }); }, From 1258a7c8a5f46f3ec5175cd74c722f8ead33c11e Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Thu, 18 Sep 2025 20:13:55 +0530 Subject: [PATCH 35/39] resolved conflict --- components/databricks/databricks.app.mjs | 190 +++++++++++++++++++++-- 1 file changed, 175 insertions(+), 15 deletions(-) diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 5f12cc346976b..1b760b96c479d 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -1,4 +1,5 @@ import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; export default { type: "app", @@ -8,14 +9,30 @@ export default { type: "string", label: "Job", description: "Identifier of a job", - async options() { - const { jobs } = await this.listJobs(); - return jobs?.map(({ + async options({ prevContext }) { + if (prevContext.pageToken === null) { + return []; + } + const { + jobs, next_page_token: pageToken, + } = await this.listJobs({ + params: { + page_token: prevContext.pageToken, + limit: constants.DEFAULT_LIMIT, + }, + }); + const options = jobs?.map(({ job_id: value, settings, }) => ({ value, - label: settings.name, + label: settings?.name || value, })) || []; + return { + options, + context: { + pageToken: pageToken || null, + }, + }; }, }, runId: { @@ -102,9 +119,9 @@ export default { }, }, methods: { - - _baseUrl() { - return `https://${this.$auth.domain}.cloud.databricks.com/api/2.0`; + getUrl(path, versionPath = constants.VERSION_PATH.V2_0) { + const baseUrl = constants.BASE_URL.replace(constants.DOMAIN_PLACEHOLDER, this.$auth.domain); + return `${baseUrl}${versionPath}${path}`; }, _headers() { return { @@ -112,12 +129,10 @@ export default { }; }, _makeRequest({ - $ = this, - path, - ...args - }) { + $ = this, path, versionPath, ...args + } = {}) { return axios($, { - url: `${this._baseUrl()}${path}`, + url: this.getUrl(path, versionPath), headers: this._headers(), paramsSerializer: { indexes: null, @@ -125,28 +140,134 @@ export default { ...args, }); }, + createJob(args = {}) { + return this._makeRequest({ + path: "/jobs/create", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, listJobs(args = {}) { return this._makeRequest({ path: "/jobs/list", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getJob(args = {}) { + return this._makeRequest({ + path: "/jobs/get", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + resetJob(args = {}) { + return this._makeRequest({ + path: "/jobs/reset", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + updateJob(args = {}) { + return this._makeRequest({ + path: "/jobs/update", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + deleteJob(args = {}) { + return this._makeRequest({ + path: "/jobs/delete", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + runJobNow(args = {}) { + return this._makeRequest({ + path: "/jobs/run-now", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/get", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, listRuns(args = {}) { return this._makeRequest({ path: "/jobs/runs/list", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + cancelRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/cancel", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + cancelAllRuns(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/cancel-all", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, getRunOutput(args = {}) { return this._makeRequest({ path: "/jobs/runs/get-output", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, - runJobNow(args = {}) { + deleteRun(args = {}) { return this._makeRequest({ - path: "/jobs/run-now", + path: "/jobs/runs/delete", method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + repairRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/repair", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + exportRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/export", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getJobPermissions({ + jobId, ...args + }) { + return this._makeRequest({ + path: `/permissions/jobs/${jobId}`, + ...args, + }); + }, + setJobPermissions({ + jobId, ...args + }) { + return this._makeRequest({ + path: `/permissions/jobs/${jobId}`, + method: "PUT", ...args, }); }, @@ -264,7 +385,6 @@ export default { ...args, }); }, - setSQLWarehousePermissions({ warehouseId, ...args }) { @@ -364,5 +484,45 @@ export default { ...args, }); }, + async paginate({ + requestor, requestorArgs = {}, + maxRequests = 3, resultsKey = "jobs", + }) { + const allResults = []; + let requestCount = 0; + let nextPageToken = null; + let hasMore = true; + + while (hasMore && requestCount < maxRequests) { + try { + const response = await requestor({ + ...requestorArgs, + params: { + ...requestorArgs.params, + page_token: nextPageToken, + }, + }); + + requestCount++; + + const results = response[resultsKey] || []; + + allResults.push(...results); + + nextPageToken = response.next_page_token; + hasMore = !!nextPageToken; + + if (results.length === 0) { + hasMore = false; + } + + } catch (error) { + console.error(`Pagination error on request ${requestCount}:`, error); + throw error; + } + } + + return allResults; + }, }, }; From 427452e5cec141f4518f05620d57ca7edadf8faf Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Thu, 18 Sep 2025 20:39:07 +0530 Subject: [PATCH 36/39] updated --- components/databricks/databricks.app.mjs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index e0c2753d7ef59..1b760b96c479d 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -148,14 +148,6 @@ export default { ...args, }); }, - createJob(args = {}) { - return this._makeRequest({ - path: "/jobs/create", - method: "POST", - versionPath: constants.VERSION_PATH.V2_2, - ...args, - }); - }, listJobs(args = {}) { return this._makeRequest({ path: "/jobs/list", From 577233e42570f6893010b53e39ef1710c8e6c83e Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Thu, 18 Sep 2025 20:55:59 +0530 Subject: [PATCH 37/39] version updated --- .../databricks/actions/cancel-all-runs/cancel-all-runs.mjs | 2 +- components/databricks/actions/cancel-run/cancel-run.mjs | 2 +- .../databricks/actions/create-endpoint/create-endpoint.mjs | 2 +- components/databricks/actions/create-job/create-job.mjs | 2 +- .../actions/create-sql-warehouse/create-sql-warehouse.mjs | 2 +- .../databricks/actions/delete-endpoint/delete-endpoint.mjs | 2 +- components/databricks/actions/delete-job/delete-job.mjs | 2 +- components/databricks/actions/delete-run/delete-run.mjs | 2 +- .../actions/delete-sql-warehouse/delete-sql-warehouse.mjs | 2 +- .../actions/edit-sql-warehouse/edit-sql-warehouse.mjs | 2 +- components/databricks/actions/export-run/export-run.mjs | 2 +- components/databricks/actions/get-endpoint/get-endpoint.mjs | 2 +- .../actions/get-job-permissions/get-job-permissions.mjs | 2 +- components/databricks/actions/get-job/get-job.mjs | 2 +- components/databricks/actions/get-run-output/get-run-output.mjs | 2 +- components/databricks/actions/get-run/get-run.mjs | 2 +- .../get-sql-warehouse-config/get-sql-warehouse-config.mjs | 2 +- .../get-sql-warehouse-permissions.mjs | 2 +- .../databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs | 2 +- components/databricks/actions/list-endpoints/list-endpoints.mjs | 2 +- components/databricks/actions/list-jobs/list-jobs.mjs | 2 +- components/databricks/actions/list-runs/list-runs.mjs | 2 +- .../actions/list-sql-warehouses/list-sql-warehouses.mjs | 2 +- components/databricks/actions/repair-run/repair-run.mjs | 2 +- components/databricks/actions/reset-job/reset-job.mjs | 2 +- components/databricks/actions/run-job-now/run-job-now.mjs | 2 +- .../actions/set-job-permissions/set-job-permissions.mjs | 2 +- .../set-sql-warehouse-config/set-sql-warehouse-config.mjs | 2 +- .../set-sql-warehouse-permissions.mjs | 2 +- .../actions/start-sql-warehouse/start-sql-warehouse.mjs | 2 +- .../actions/stop-sql-warehouse/stop-sql-warehouse.mjs | 2 +- components/databricks/actions/update-job/update-job.mjs | 2 +- components/databricks/package.json | 2 +- 33 files changed, 33 insertions(+), 33 deletions(-) diff --git a/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs index 592740cc949fd..6cbec9bd8ea1c 100644 --- a/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs +++ b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-cancel-all-runs", name: "Cancel All Runs", description: "Cancel all active runs for a job. The runs are canceled asynchronously, so it doesn't prevent new runs from being started. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelallruns)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/cancel-run/cancel-run.mjs b/components/databricks/actions/cancel-run/cancel-run.mjs index 9af5f0fd4c76a..bb54ba37e5d37 100644 --- a/components/databricks/actions/cancel-run/cancel-run.mjs +++ b/components/databricks/actions/cancel-run/cancel-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-cancel-run", name: "Cancel Run", description: "Cancel a job run. The run is canceled asynchronously, so it may still be running when this request completes. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/create-endpoint/create-endpoint.mjs b/components/databricks/actions/create-endpoint/create-endpoint.mjs index 33bf7ad4d7a8f..046e54bc31dd1 100644 --- a/components/databricks/actions/create-endpoint/create-endpoint.mjs +++ b/components/databricks/actions/create-endpoint/create-endpoint.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-endpoint", name: "Create Endpoint", description: "Create a new vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/createendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-job/create-job.mjs b/components/databricks/actions/create-job/create-job.mjs index 209cf119ac164..708146085ae73 100644 --- a/components/databricks/actions/create-job/create-job.mjs +++ b/components/databricks/actions/create-job/create-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-job", name: "Create Job", description: "Create a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/create)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index b953d78090e61..02de4ce9f5c39 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs index 4a95bb6f13e2d..6750820463af8 100644 --- a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs +++ b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-endpoint", name: "Delete Endpoint", description: "Delete a vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/deleteendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-job/delete-job.mjs b/components/databricks/actions/delete-job/delete-job.mjs index a5e55b716421b..79b6c528ee93e 100644 --- a/components/databricks/actions/delete-job/delete-job.mjs +++ b/components/databricks/actions/delete-job/delete-job.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-job", name: "Delete Job", description: "Delete a job. Deleted jobs cannot be recovered. [See the documentation](https://docs.databricks.com/api/workspace/jobs/delete)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/delete-run/delete-run.mjs b/components/databricks/actions/delete-run/delete-run.mjs index 2fbda0185965d..b27836b1cabb5 100644 --- a/components/databricks/actions/delete-run/delete-run.mjs +++ b/components/databricks/actions/delete-run/delete-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-run", name: "Delete Run", description: "Delete a non-active run. Returns an error if the run is active. [See the documentation](https://docs.databricks.com/api/workspace/jobs/deleterun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 47584608434df..3300848ba3852 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-sql-warehouse", name: "Delete SQL Warehouse", description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index 293ee437f120a..b98292e6a5faa 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/export-run/export-run.mjs b/components/databricks/actions/export-run/export-run.mjs index 63fca03ae7303..71e8e28fd7ea9 100644 --- a/components/databricks/actions/export-run/export-run.mjs +++ b/components/databricks/actions/export-run/export-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-export-run", name: "Export Run", description: "Export and retrieve the job run task. [See the documentation](https://docs.databricks.com/api/workspace/jobs/exportrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-endpoint/get-endpoint.mjs b/components/databricks/actions/get-endpoint/get-endpoint.mjs index 1f494f4658be1..0937ff8482688 100644 --- a/components/databricks/actions/get-endpoint/get-endpoint.mjs +++ b/components/databricks/actions/get-endpoint/get-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-endpoint", name: "Get Endpoint", description: "Get details of a specific vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/getendpoint)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-job-permissions/get-job-permissions.mjs b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs index 3c9bae4657b9d..c1e77b0564888 100644 --- a/components/databricks/actions/get-job-permissions/get-job-permissions.mjs +++ b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-job-permissions", name: "Get Job Permissions", description: "Get permissions of a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getpermissions)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-job/get-job.mjs b/components/databricks/actions/get-job/get-job.mjs index 6e418684cdd14..fd029c564e3b8 100644 --- a/components/databricks/actions/get-job/get-job.mjs +++ b/components/databricks/actions/get-job/get-job.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-job", name: "Get Job", description: "Retrieves the details for a single job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/get)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-run-output/get-run-output.mjs b/components/databricks/actions/get-run-output/get-run-output.mjs index c44b1001a7bfa..6bc6ee495e0cb 100644 --- a/components/databricks/actions/get-run-output/get-run-output.mjs +++ b/components/databricks/actions/get-run-output/get-run-output.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run-output", name: "Get Run Output", description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-run/get-run.mjs b/components/databricks/actions/get-run/get-run.mjs index 84d95be7c0d5c..607fee72566b8 100644 --- a/components/databricks/actions/get-run/get-run.mjs +++ b/components/databricks/actions/get-run/get-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run", name: "Get Run", description: "Retrieve the metadata of a run. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index 1fbdda709e71b..3ab986eb06a7a 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", description: "Retrieves the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 4d76f5bb41c74..c7098db9d4156 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", description: "Retrieves the permissions for a specific SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 28d131c05c8f1..21fd9701333a9 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse", name: "Get SQL Warehouse", description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/get)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-endpoints/list-endpoints.mjs b/components/databricks/actions/list-endpoints/list-endpoints.mjs index c0fe92e1672e6..9e692d7ecdae7 100644 --- a/components/databricks/actions/list-endpoints/list-endpoints.mjs +++ b/components/databricks/actions/list-endpoints/list-endpoints.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-endpoints", name: "List Endpoints", description: "List all vector search endpoints. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/listendpoints)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-jobs/list-jobs.mjs b/components/databricks/actions/list-jobs/list-jobs.mjs index 9f60a6ea0224b..4781153909c4b 100644 --- a/components/databricks/actions/list-jobs/list-jobs.mjs +++ b/components/databricks/actions/list-jobs/list-jobs.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-list-jobs", name: "List Jobs", description: "List all jobs using automatic pagination. [See the documentation](https://docs.databricks.com/api/workspace/jobs/list)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/list-runs/list-runs.mjs b/components/databricks/actions/list-runs/list-runs.mjs index d2d7fe17cd3f3..e03846d411551 100644 --- a/components/databricks/actions/list-runs/list-runs.mjs +++ b/components/databricks/actions/list-runs/list-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-runs", name: "List Runs", description: "Lists all runs available to the user. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index 356622572a9a9..5b63c9c874a82 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-sql-warehouses", name: "List SQL Warehouses", description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/list)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/repair-run/repair-run.mjs b/components/databricks/actions/repair-run/repair-run.mjs index e5f9234fdf800..ef4e4015fb5db 100644 --- a/components/databricks/actions/repair-run/repair-run.mjs +++ b/components/databricks/actions/repair-run/repair-run.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-repair-run", name: "Repair Run", description: "Re-run one or more tasks. [See the documentation](https://docs.databricks.com/api/workspace/jobs/repairrun)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/reset-job/reset-job.mjs b/components/databricks/actions/reset-job/reset-job.mjs index cf6acbc2af856..44aa051ea1d3f 100644 --- a/components/databricks/actions/reset-job/reset-job.mjs +++ b/components/databricks/actions/reset-job/reset-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-reset-job", name: "Reset Job", description: "Overwrite all settings for the given job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/reset)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/run-job-now/run-job-now.mjs b/components/databricks/actions/run-job-now/run-job-now.mjs index 4318cb98d9e8a..11c6e52ad930c 100644 --- a/components/databricks/actions/run-job-now/run-job-now.mjs +++ b/components/databricks/actions/run-job-now/run-job-now.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-run-job-now", name: "Run Job Now", description: "Run a job now and return the id of the triggered run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.4", + version: "0.0.5", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-job-permissions/set-job-permissions.mjs b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs index adf2897ec8faf..5c0cde44682c1 100644 --- a/components/databricks/actions/set-job-permissions/set-job-permissions.mjs +++ b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-set-job-permissions", name: "Set Job Permissions", description: "Set permissions on a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/setpermissions)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index 6c9d4a7b98e6e..c8e1252417614 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", description: "Updates the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index 202d1677dd5a9..9d62bd131465b 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-permissions", name: "Set SQL Warehouse Permissions", description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/setpermissions)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 669a593323040..5cba5641b2bd3 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-start-sql-warehouse", name: "Start SQL Warehouse", description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/start)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index 0117fca5d801d..8c1b8a932a0a8 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-stop-sql-warehouse", name: "Stop SQL Warehouse", description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/stop)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/update-job/update-job.mjs b/components/databricks/actions/update-job/update-job.mjs index 6d6afec50bb7f..8cb8f6c81d36d 100644 --- a/components/databricks/actions/update-job/update-job.mjs +++ b/components/databricks/actions/update-job/update-job.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-update-job", name: "Update Job", description: "Update an existing job. Only the fields that are provided will be updated. [See the documentation](https://docs.databricks.com/api/workspace/jobs/update)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { app, diff --git a/components/databricks/package.json b/components/databricks/package.json index c82b72576579b..22ed3bf6d6cb6 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.4.0", + "version": "0.5.0", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [ From 8df764b8c0cb23c6415d256f0779bd7ed00bbff4 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Fri, 19 Sep 2025 16:16:40 +0530 Subject: [PATCH 38/39] updated --- components/databricks/databricks.app.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 1b760b96c479d..607764fdc69e4 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -134,9 +134,6 @@ export default { return axios($, { url: this.getUrl(path, versionPath), headers: this._headers(), - paramsSerializer: { - indexes: null, - }, ...args, }); }, @@ -461,6 +458,9 @@ export default { path: `/vector-search/indexes/${indexName}/delete-data`, method: "DELETE", params, + paramsSerializer: { + indexes: null, + }, ...args, }); }, From db3b6cd67685fbe1fb1f47fda9b5d1eeb01c3fa3 Mon Sep 17 00:00:00 2001 From: Lokesh chand Date: Sun, 21 Sep 2025 17:38:18 +0530 Subject: [PATCH 39/39] updated --- .../upsert-vector-search-index-data.mjs | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs index c3134d9f1a8ba..6e4f5cca592e9 100644 --- a/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs +++ b/components/databricks/actions/upsert-vector-search-index-data/upsert-vector-search-index-data.mjs @@ -26,9 +26,9 @@ export default { ], }, rows: { - type: "string[]", + type: "string", label: "Rows to Upsert", - description: "Array of rows to upsert. Each row should be a JSON object string. Example: `{ \"id\": \"1\", \"text\": \"hello world\", \"text_vector\": [1.0, 2.0, 3.0] }`", + description: "Array of rows to upsert. Each row should be a JSON object string. Example: `[{ \"id\": \"1\", \"text\": \"hello world\", \"text_vector\": [0.1, 0.2, 0.3] }]`", }, }, @@ -36,25 +36,14 @@ export default { const parsedRows = utils.parseObject(this.rows); if (!Array.isArray(parsedRows) || !parsedRows.length) { - throw new ConfigurationError("rows must be a non-empty array of JSON objects."); + throw new ConfigurationError("rows must be a non-empty JSON array."); } - parsedRows.forEach((row, idx) => { - if (!row || typeof row !== "object") { - throw new ConfigurationError(`Row at index ${idx} is invalid. Each row must be a JSON object.`); - } - if (!row.id) { - throw new ConfigurationError(`Row at index ${idx} is missing required primary key field "id".`); - } - }); - - const payload = { - index_name: this.indexName, - inputs_json: JSON.stringify(parsedRows), - }; - const response = await this.databricks.upsertVectorSearchIndexData({ - data: payload, + indexName: this.indexName, + data: { + inputs_json: JSON.stringify(parsedRows), + }, $, });