diff --git a/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs new file mode 100644 index 0000000000000..592740cc949fd --- /dev/null +++ b/components/databricks/actions/cancel-all-runs/cancel-all-runs.mjs @@ -0,0 +1,52 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-cancel-all-runs", + name: "Cancel All Runs", + description: "Cancel all active runs for a job. The runs are canceled asynchronously, so it doesn't prevent new runs from being started. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelallruns)", + version: "0.0.1", + type: "action", + props: { + app, + // eslint-disable-next-line pipedream/props-label, pipedream/props-description + info: { + type: "alert", + alertType: "info", + content: "Either a **Job** or **All Queued Runs** must be provided.", + }, + jobId: { + optional: true, + propDefinition: [ + app, + "jobId", + ], + }, + allQueuedRuns: { + type: "boolean", + label: "All Queued Runs", + description: "Optional boolean parameter to cancel all queued runs. If no **Job ID** is provided, all queued runs in the workspace are canceled.", + optional: true, + }, + }, + async run({ $ }) { + const { + app, + jobId, + allQueuedRuns, + } = this; + + await app.cancelAllRuns({ + $, + data: { + job_id: jobId, + all_queued_runs: allQueuedRuns, + }, + }); + + $.export("$summary", "Successfully initiated cancellation of all runs"); + + return { + success: true, + }; + }, +}; diff --git a/components/databricks/actions/cancel-run/cancel-run.mjs b/components/databricks/actions/cancel-run/cancel-run.mjs new file mode 100644 index 0000000000000..9af5f0fd4c76a --- /dev/null +++ b/components/databricks/actions/cancel-run/cancel-run.mjs @@ -0,0 +1,37 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-cancel-run", + name: "Cancel Run", + description: "Cancel a job run. The run is canceled asynchronously, so it may still be running when this request completes. [See the documentation](https://docs.databricks.com/api/workspace/jobs/cancelrun)", + version: "0.0.1", + type: "action", + props: { + app, + runId: { + propDefinition: [ + app, + "runId", + ], + }, + }, + async run({ $ }) { + const { + app, + runId, + } = this; + + await app.cancelRun({ + $, + data: { + run_id: runId, + }, + }); + + $.export("$summary", `Successfully initiated cancellation of run with ID \`${runId}\`.`); + + return { + success: true, + }; + }, +}; diff --git a/components/databricks/actions/create-endpoint/create-endpoint.mjs b/components/databricks/actions/create-endpoint/create-endpoint.mjs index 3777a7ca32c5e..33bf7ad4d7a8f 100644 --- a/components/databricks/actions/create-endpoint/create-endpoint.mjs +++ b/components/databricks/actions/create-endpoint/create-endpoint.mjs @@ -5,7 +5,7 @@ export default { key: "databricks-create-endpoint", name: "Create Endpoint", description: "Create a new vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/createendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/create-job/create-job.mjs b/components/databricks/actions/create-job/create-job.mjs new file mode 100644 index 0000000000000..209cf119ac164 --- /dev/null +++ b/components/databricks/actions/create-job/create-job.mjs @@ -0,0 +1,237 @@ +import app from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-create-job", + name: "Create Job", + description: "Create a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/create)", + version: "0.0.1", + type: "action", + props: { + app, + tasks: { + type: "string[]", + label: "Tasks", + description: `A list of task specifications to be executed by this job. JSON string format. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#tasks) for task specification details. + +**Example:** +\`\`\`json +[ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/sharky@databricks.com/weather_ingest" + }, + "task_key": "weather_ocean_data" + } +] +\`\`\` + `, + }, + name: { + type: "string", + label: "Job Name", + description: "An optional name for the job", + optional: true, + }, + tags: { + type: "object", + label: "Tags", + description: "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags", + optional: true, + }, + jobClusters: { + type: "string[]", + label: "Job Clusters", + description: `A list of job cluster specifications that can be shared and reused by tasks of this job. JSON string format. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#job_clusters) for job cluster specification details. + +**Example:** +\`\`\`json +[ + { + "job_cluster_key": "auto_scaling_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 16, + "min_workers": 2 + }, + "node_type_id": null, + "spark_conf": { + "spark.speculation": true + }, + "spark_version": "7.3.x-scala2.12" + } + } +] +\`\`\` + `, + optional: true, + }, + emailNotifications: { + type: "string", + label: "Email Notifications", + description: `An optional set of email addresses to notify when runs of this job begin, complete, or when the job is deleted. Specify as a JSON object with keys for each notification type. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#email_notifications) for details on each field. + +**Example:** +\`\`\`json +{ + "on_start": ["user1@example.com"], + "on_success": ["user2@example.com"], + "on_failure": ["user3@example.com"], + "on_duration_warning_threshold_exceeded": ["user4@example.com"], + "on_streaming_backlog_exceeded": ["user5@example.com"] +} +\`\`\` +`, + optional: true, + }, + webhookNotifications: { + type: "string", + label: "Webhook Notifications", + description: `A collection of system notification IDs to notify when runs of this job begin, complete, or encounter specific events. Specify as a JSON object with keys for each notification type. Each key accepts an array of objects with an \`id\` property (system notification ID). A maximum of 3 destinations can be specified for each property. + +Supported keys: +- \`on_start\`: Notified when the run starts. +- \`on_success\`: Notified when the run completes successfully. +- \`on_failure\`: Notified when the run fails. +- \`on_duration_warning_threshold_exceeded\`: Notified when the run duration exceeds the specified threshold. +- \`on_streaming_backlog_exceeded\`: Notified when streaming backlog thresholds are exceeded. + +[See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#webhook_notifications) for details. + +**Example:** +\`\`\`json +{ + "on_success": [ + { "id": "https://eoiqkb8yzox6u2n.m.pipedream.net" } + ], + "on_failure": [ + { "id": "https://another-webhook-url.com/notify" } + ] +} +\`\`\` +`, + optional: true, + }, + timeoutSeconds: { + type: "integer", + label: "Timeout Seconds", + description: "An optional timeout applied to each run of this job. The default behavior is to have no timeout", + optional: true, + }, + schedule: { + type: "string", + label: "Schedule", + description: `An optional periodic schedule for this job, specified as a JSON object. By default, the job only runs when triggered manually or via the API. The schedule object must include: + +- \`quartz_cron_expression\` (**required**): A Cron expression using Quartz syntax that defines when the job runs. [See Cron Trigger details](https://docs.databricks.com/api/workspace/jobs/create#schedule). +- \`timezone_id\` (**required**): A Java timezone ID (e.g., "Europe/London") that determines the timezone for the schedule. [See Java TimeZone details](https://docs.databricks.com/api/workspace/jobs/create#schedule). +- \`pause_status\` (optional): Set to \`"UNPAUSED"\` (default) or \`"PAUSED"\` to control whether the schedule is active. + +**Example:** +\`\`\`json +{ + "quartz_cron_expression": "0 0 12 * * ?", + "timezone_id": "Asia/Ho_Chi_Minh", + "pause_status": "UNPAUSED" +} +\`\`\` +`, + optional: true, + }, + maxConcurrentRuns: { + type: "integer", + label: "Max Concurrent Runs", + description: "An optional maximum allowed number of concurrent runs of the job. Defaults to 1", + optional: true, + }, + gitSource: { + type: "string", + label: "Git Source", + description: `An optional specification for a remote Git repository containing the source code used by tasks. Provide as a JSON string. + +This enables version-controlled source code for notebook, dbt, Python script, and SQL File tasks. If \`git_source\` is set, these tasks retrieve files from the remote repository by default (can be overridden per task by setting \`source\` to \`WORKSPACE\`). **Note:** dbt and SQL File tasks require \`git_source\` to be defined. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#git_source) for more details. + +**Fields:** +- \`git_url\` (**required**): URL of the repository to be cloned (e.g., "https://github.com/databricks/databricks-cli"). +- \`git_provider\` (**required**): Service hosting the repository. One of: \`gitHub\`, \`bitbucketCloud\`, \`azureDevOpsServices\`, \`gitHubEnterprise\`, \`bitbucketServer\`, \`gitLab\`, \`gitLabEnterpriseEdition\`, \`awsCodeCommit\`. +- \`git_branch\`: Name of the branch to check out (cannot be used with \`git_tag\` or \`git_commit\`). +- \`git_tag\`: Name of the tag to check out (cannot be used with \`git_branch\` or \`git_commit\`). +- \`git_commit\`: Commit hash to check out (cannot be used with \`git_branch\` or \`git_tag\`). + +**Example:** +\`\`\`json +{ + "git_url": "https://github.com/databricks/databricks-cli", + "git_provider": "gitHub", + "git_branch": "main" +} +\`\`\` +`, + optional: true, + }, + accessControlList: { + type: "string[]", + label: "Access Control List", + description: `A list of permissions to set on the job, specified as a JSON array of objects. Each object can define permissions for a user, group, or service principal. + +Each object may include: +- \`user_name\`: Name of the user. +- \`group_name\`: Name of the group. +- \`service_principal_name\`: Application ID of a service principal. +- \`permission_level\`: Permission level. One of: \`CAN_MANAGE\`, \`IS_OWNER\`, \`CAN_MANAGE_RUN\`, \`CAN_VIEW\`. + +**Example:** +\`\`\`json +[ + { + "permission_level": "IS_OWNER", + "user_name": "jorge.c@turing.com" + }, + { + "permission_level": "CAN_VIEW", + "group_name": "data-scientists" + } +] +\`\`\` +[See the API documentation](https://docs.databricks.com/api/workspace/jobs/create#access_control_list) for more details.`, + optional: true, + }, + }, + async run({ $ }) { + const { + app, + tasks, + name, + tags, + jobClusters, + emailNotifications, + webhookNotifications, + timeoutSeconds, + schedule, + maxConcurrentRuns, + gitSource, + accessControlList, + } = this; + + const response = await app.createJob({ + $, + data: { + name, + tags, + tasks: utils.parseJsonInput(tasks), + job_clusters: utils.parseJsonInput(jobClusters), + email_notifications: utils.parseJsonInput(emailNotifications), + webhook_notifications: utils.parseJsonInput(webhookNotifications), + timeout_seconds: timeoutSeconds, + schedule: utils.parseJsonInput(schedule), + max_concurrent_runs: maxConcurrentRuns, + git_source: utils.parseJsonInput(gitSource), + access_control_list: utils.parseJsonInput(accessControlList), + }, + }); + + $.export("$summary", `Successfully created job with ID \`${response.job_id}\``); + + return response; + }, +}; diff --git a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs index d0552107d1101..b953d78090e61 100644 --- a/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs +++ b/components/databricks/actions/create-sql-warehouse/create-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-create-sql-warehouse", name: "Create SQL Warehouse", description: "Creates a new SQL Warehouse in Databricks. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/create)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs index 5ad2a8c43304f..4a95bb6f13e2d 100644 --- a/components/databricks/actions/delete-endpoint/delete-endpoint.mjs +++ b/components/databricks/actions/delete-endpoint/delete-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-endpoint", name: "Delete Endpoint", description: "Delete a vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/deleteendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/delete-job/delete-job.mjs b/components/databricks/actions/delete-job/delete-job.mjs new file mode 100644 index 0000000000000..a5e55b716421b --- /dev/null +++ b/components/databricks/actions/delete-job/delete-job.mjs @@ -0,0 +1,37 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-job", + name: "Delete Job", + description: "Delete a job. Deleted jobs cannot be recovered. [See the documentation](https://docs.databricks.com/api/workspace/jobs/delete)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + }, + async run({ $ }) { + const { + app, + jobId, + } = this; + + await app.deleteJob({ + $, + data: { + job_id: jobId, + }, + }); + + $.export("$summary", `Successfully deleted job with ID \`${jobId}\`.`); + + return { + success: true, + }; + }, +}; diff --git a/components/databricks/actions/delete-run/delete-run.mjs b/components/databricks/actions/delete-run/delete-run.mjs new file mode 100644 index 0000000000000..2fbda0185965d --- /dev/null +++ b/components/databricks/actions/delete-run/delete-run.mjs @@ -0,0 +1,30 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-delete-run", + name: "Delete Run", + description: "Delete a non-active run. Returns an error if the run is active. [See the documentation](https://docs.databricks.com/api/workspace/jobs/deleterun)", + version: "0.0.1", + type: "action", + props: { + app, + runId: { + propDefinition: [ + app, + "runId", + ], + }, + }, + async run({ $ }) { + const response = await this.app.deleteRun({ + data: { + run_id: this.runId, + }, + $, + }); + + $.export("$summary", `Successfully deleted run with ID ${this.runId}.`); + + return response || {}; + }, +}; diff --git a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs index 1004fe38fd410..47584608434df 100644 --- a/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs +++ b/components/databricks/actions/delete-sql-warehouse/delete-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-delete-sql-warehouse", name: "Delete SQL Warehouse", description: "Deletes a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/delete)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs index ff28b9a5eea8f..293ee437f120a 100644 --- a/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs +++ b/components/databricks/actions/edit-sql-warehouse/edit-sql-warehouse.mjs @@ -7,7 +7,7 @@ export default { key: "databricks-edit-sql-warehouse", name: "Edit SQL Warehouse", description: "Edits the configuration of an existing SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/edit)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/export-run/export-run.mjs b/components/databricks/actions/export-run/export-run.mjs new file mode 100644 index 0000000000000..63fca03ae7303 --- /dev/null +++ b/components/databricks/actions/export-run/export-run.mjs @@ -0,0 +1,48 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-export-run", + name: "Export Run", + description: "Export and retrieve the job run task. [See the documentation](https://docs.databricks.com/api/workspace/jobs/exportrun)", + version: "0.0.1", + type: "action", + props: { + app, + runId: { + propDefinition: [ + app, + "runId", + ], + }, + viewsToExport: { + type: "string", + label: "Views to Export", + description: "Which views to export. Defaults to `CODE`", + optional: true, + options: [ + "CODE", + "DASHBOARDS", + "ALL", + ], + }, + }, + async run({ $ }) { + const { + app, + runId, + viewsToExport, + } = this; + + const response = await app.exportRun({ + $, + params: { + run_id: runId, + views_to_export: viewsToExport, + }, + }); + + $.export("$summary", `Successfully exported run with ID \`${runId}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/get-endpoint/get-endpoint.mjs b/components/databricks/actions/get-endpoint/get-endpoint.mjs index 99fd6b71abb96..1f494f4658be1 100644 --- a/components/databricks/actions/get-endpoint/get-endpoint.mjs +++ b/components/databricks/actions/get-endpoint/get-endpoint.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-endpoint", name: "Get Endpoint", description: "Get details of a specific vector search endpoint. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/getendpoint)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-job-permissions/get-job-permissions.mjs b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs new file mode 100644 index 0000000000000..3c9bae4657b9d --- /dev/null +++ b/components/databricks/actions/get-job-permissions/get-job-permissions.mjs @@ -0,0 +1,33 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-job-permissions", + name: "Get Job Permissions", + description: "Get permissions of a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getpermissions)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + }, + async run({ $ }) { + const { + app, + jobId, + } = this; + + const response = await app.getJobPermissions({ + $, + jobId, + }); + + $.export("$summary", `Successfully retrieved permissions for job with ID \`${jobId}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/get-job/get-job.mjs b/components/databricks/actions/get-job/get-job.mjs new file mode 100644 index 0000000000000..6e418684cdd14 --- /dev/null +++ b/components/databricks/actions/get-job/get-job.mjs @@ -0,0 +1,35 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-job", + name: "Get Job", + description: "Retrieves the details for a single job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/get)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + }, + async run({ $ }) { + const { + app, + jobId, + } = this; + + const response = await app.getJob({ + $, + params: { + job_id: jobId, + }, + }); + + $.export("$summary", `Successfully retrieved job with ID \`${response.job_id}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/get-run-output/get-run-output.mjs b/components/databricks/actions/get-run-output/get-run-output.mjs index 2d2434c1c284b..c44b1001a7bfa 100644 --- a/components/databricks/actions/get-run-output/get-run-output.mjs +++ b/components/databricks/actions/get-run-output/get-run-output.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-run-output", name: "Get Run Output", description: "Retrieve the output and metadata of a single task run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-get-output)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-run/get-run.mjs b/components/databricks/actions/get-run/get-run.mjs new file mode 100644 index 0000000000000..84d95be7c0d5c --- /dev/null +++ b/components/databricks/actions/get-run/get-run.mjs @@ -0,0 +1,51 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-get-run", + name: "Get Run", + description: "Retrieve the metadata of a run. [See the documentation](https://docs.databricks.com/api/workspace/jobs/getrun)", + version: "0.0.1", + type: "action", + props: { + app, + runId: { + propDefinition: [ + app, + "runId", + ], + }, + includeHistory: { + type: "boolean", + label: "Include History", + description: "Whether to include the repair history in the response", + optional: true, + }, + includeResolvedValues: { + type: "boolean", + label: "Include Resolved Values", + description: "Whether to include resolved parameter values in the response", + optional: true, + }, + }, + async run({ $ }) { + const { + app, + runId, + includeHistory, + includeResolvedValues, + } = this; + + const response = await app.getRun({ + $, + params: { + run_id: runId, + include_history: includeHistory, + include_resolved_values: includeResolvedValues, + }, + }); + + $.export("$summary", `Successfully retrieved run with ID \`${response.job_run_id}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs index 0cdfa6a1129dc..1fbdda709e71b 100644 --- a/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs +++ b/components/databricks/actions/get-sql-warehouse-config/get-sql-warehouse-config.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-config", name: "Get SQL Warehouse Config", description: "Retrieves the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getworkspacewarehouseconfig)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs index 8c3085c52cd73..4d76f5bb41c74 100644 --- a/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/get-sql-warehouse-permissions/get-sql-warehouse-permissions.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse-permissions", name: "Get SQL Warehouse Permissions", description: "Retrieves the permissions for a specific SQL Warehouse. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/getpermissions)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs index 8476e09bb8a65..28d131c05c8f1 100644 --- a/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs +++ b/components/databricks/actions/get-sql-warehouse/get-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-get-sql-warehouse", name: "Get SQL Warehouse", description: "Retrieves details for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/get)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-endpoints/list-endpoints.mjs b/components/databricks/actions/list-endpoints/list-endpoints.mjs index ed06bdebf9710..c0fe92e1672e6 100644 --- a/components/databricks/actions/list-endpoints/list-endpoints.mjs +++ b/components/databricks/actions/list-endpoints/list-endpoints.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-endpoints", name: "List Endpoints", description: "List all vector search endpoints. [See the documentation](https://docs.databricks.com/api/workspace/vectorsearchendpoints/listendpoints)", - version: "0.0.1", + version: "0.0.2", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-jobs/list-jobs.mjs b/components/databricks/actions/list-jobs/list-jobs.mjs new file mode 100644 index 0000000000000..9f60a6ea0224b --- /dev/null +++ b/components/databricks/actions/list-jobs/list-jobs.mjs @@ -0,0 +1,59 @@ +import app from "../../databricks.app.mjs"; +import constants from "../../common/constants.mjs"; + +export default { + key: "databricks-list-jobs", + name: "List Jobs", + description: "List all jobs using automatic pagination. [See the documentation](https://docs.databricks.com/api/workspace/jobs/list)", + version: "0.0.1", + type: "action", + props: { + app, + expandTasks: { + type: "boolean", + label: "Expand Tasks", + description: "Whether to include task and cluster details in the response", + optional: true, + }, + name: { + type: "string", + label: "Job Name", + description: "Optional name to filter on", + optional: true, + }, + maxRequests: { + type: "integer", + label: "Max Requests", + description: "Maximum number of API requests to make when paginating", + optional: true, + min: 1, + max: 10, + }, + }, + async run({ $ }) { + const { + app, + expandTasks, + name, + maxRequests, + } = this; + + const jobs = await app.paginate({ + requestor: app.listJobs, + maxRequests, + resultsKey: "jobs", + requestorArgs: { + $, + params: { + expand_tasks: expandTasks, + name, + limit: constants.DEFAULT_LIMIT, + }, + }, + }); + + $.export("$summary", `Successfully retrieved \`${jobs.length}\` job(s)`); + + return jobs; + }, +}; diff --git a/components/databricks/actions/list-runs/list-runs.mjs b/components/databricks/actions/list-runs/list-runs.mjs index 534eee7649c1b..d2d7fe17cd3f3 100644 --- a/components/databricks/actions/list-runs/list-runs.mjs +++ b/components/databricks/actions/list-runs/list-runs.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-runs", name: "List Runs", description: "Lists all runs available to the user. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs index f83290f306ca4..356622572a9a9 100644 --- a/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs +++ b/components/databricks/actions/list-sql-warehouses/list-sql-warehouses.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-list-sql-warehouses", name: "List SQL Warehouses", description: "Lists all SQL Warehouses available in the Databricks workspace. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/list)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/repair-run/repair-run.mjs b/components/databricks/actions/repair-run/repair-run.mjs new file mode 100644 index 0000000000000..e5f9234fdf800 --- /dev/null +++ b/components/databricks/actions/repair-run/repair-run.mjs @@ -0,0 +1,67 @@ +import app from "../../databricks.app.mjs"; + +export default { + key: "databricks-repair-run", + name: "Repair Run", + description: "Re-run one or more tasks. [See the documentation](https://docs.databricks.com/api/workspace/jobs/repairrun)", + version: "0.0.1", + type: "action", + props: { + app, + // eslint-disable-next-line pipedream/props-label, pipedream/props-description + info: { + type: "alert", + alertType: "info", + content: "Either a **Rerun Tasks** or **Rerun All Failed Tasks** must be provided.", + }, + runId: { + propDefinition: [ + app, + "runId", + ], + }, + rerunTasks: { + type: "string[]", + label: "Rerun Tasks", + description: "The task keys of the task runs to repair", + optional: true, + }, + rerunAllFailedTasks: { + type: "boolean", + label: "Rerun All Failed Tasks", + description: "If true, repair all failed tasks. Only one of rerun_tasks or rerun_all_failed_tasks can be used", + optional: true, + }, + pipelineParamsFullRefresh: { + type: "boolean", + label: "Pipeline Params - Full Refresh", + description: "Controls whether the pipeline should perform a full refresh", + optional: true, + }, + }, + async run({ $ }) { + const { + app, + runId, + rerunTasks, + rerunAllFailedTasks, + pipelineParamsFullRefresh, + } = this; + + const response = await app.repairRun({ + $, + data: { + run_id: runId, + rerun_tasks: rerunTasks, + rerun_all_failed_tasks: rerunAllFailedTasks, + pipeline_params: { + full_refresh: pipelineParamsFullRefresh, + }, + }, + }); + + $.export("$summary", `Successfully initiated repair of run with ID \`${response.repair_id}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/reset-job/reset-job.mjs b/components/databricks/actions/reset-job/reset-job.mjs new file mode 100644 index 0000000000000..cf6acbc2af856 --- /dev/null +++ b/components/databricks/actions/reset-job/reset-job.mjs @@ -0,0 +1,45 @@ +import app from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-reset-job", + name: "Reset Job", + description: "Overwrite all settings for the given job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/reset)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + newSettings: { + type: "string", + label: "New Settings", + description: "The new settings for the job. JSON string format with the complete job specification. [See the documentation](https://docs.databricks.com/api/workspace/jobs/reset#new_settings)", + }, + }, + async run({ $ }) { + const { + app, + jobId, + newSettings, + } = this; + + await app.resetJob({ + $, + data: { + job_id: jobId, + new_settings: utils.parseJsonInput(newSettings), + }, + }); + + $.export("$summary", `Successfully reset job with ID \`${jobId}\`.`); + + return { + success: true, + }; + }, +}; diff --git a/components/databricks/actions/run-job-now/run-job-now.mjs b/components/databricks/actions/run-job-now/run-job-now.mjs index 046f6dfb6c8bb..4318cb98d9e8a 100644 --- a/components/databricks/actions/run-job-now/run-job-now.mjs +++ b/components/databricks/actions/run-job-now/run-job-now.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-run-job-now", name: "Run Job Now", description: "Run a job now and return the id of the triggered run. [See the documentation](https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runs-list)", - version: "0.0.3", + version: "0.0.4", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-job-permissions/set-job-permissions.mjs b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs new file mode 100644 index 0000000000000..adf2897ec8faf --- /dev/null +++ b/components/databricks/actions/set-job-permissions/set-job-permissions.mjs @@ -0,0 +1,54 @@ +import app from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-set-job-permissions", + name: "Set Job Permissions", + description: "Set permissions on a job. [See the documentation](https://docs.databricks.com/api/workspace/jobs/setpermissions)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + accessControlList: { + type: "string[]", + label: "Access Control List", + description: `List of permissions to set on the job. JSON string format with array of permission objects. [See the documentation](https://docs.databricks.com/api/workspace/jobs/setpermissions#access_control_list) + +**Example:** +\`\`\`json +[ + { + "permission_level": "IS_OWNER", + "user_name": "user@example.com" + } +] +\`\`\` +`, + }, + }, + async run({ $ }) { + const { + app, + jobId, + accessControlList, + } = this; + + const response = await app.setJobPermissions({ + $, + jobId, + data: { + access_control_list: utils.parseJsonInput(accessControlList), + }, + }); + + $.export("$summary", `Successfully set permissions for job with ID \`${jobId}\`.`); + + return response; + }, +}; diff --git a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs index dddc5e9f40ddf..6c9d4a7b98e6e 100644 --- a/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs +++ b/components/databricks/actions/set-sql-warehouse-config/set-sql-warehouse-config.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-config", name: "Set SQL Warehouse Config", description: "Updates the global configuration for SQL Warehouses. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/setworkspacewarehouseconfig)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs index ff609e74645f5..202d1677dd5a9 100644 --- a/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs +++ b/components/databricks/actions/set-sql-warehouse-permissions/set-sql-warehouse-permissions.mjs @@ -6,7 +6,7 @@ export default { key: "databricks-set-sql-warehouse-permissions", name: "Set SQL Warehouse Permissions", description: "Updates the permissions for a specific SQL Warehouse. [See docs](https://docs.databricks.com/api/workspace/warehouses/setpermissions)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs index 41b127c62a4d7..669a593323040 100644 --- a/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs +++ b/components/databricks/actions/start-sql-warehouse/start-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-start-sql-warehouse", name: "Start SQL Warehouse", description: "Starts a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/start)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs index cffe2875a1540..0117fca5d801d 100644 --- a/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs +++ b/components/databricks/actions/stop-sql-warehouse/stop-sql-warehouse.mjs @@ -4,7 +4,7 @@ export default { key: "databricks-stop-sql-warehouse", name: "Stop SQL Warehouse", description: "Stops a SQL Warehouse by ID. [See the documentation](https://docs.databricks.com/api/workspace/warehouses/stop)", - version: "0.0.2", + version: "0.0.3", type: "action", props: { databricks, diff --git a/components/databricks/actions/update-job/update-job.mjs b/components/databricks/actions/update-job/update-job.mjs new file mode 100644 index 0000000000000..6d6afec50bb7f --- /dev/null +++ b/components/databricks/actions/update-job/update-job.mjs @@ -0,0 +1,62 @@ +import app from "../../databricks.app.mjs"; +import utils from "../../common/utils.mjs"; + +export default { + key: "databricks-update-job", + name: "Update Job", + description: "Update an existing job. Only the fields that are provided will be updated. [See the documentation](https://docs.databricks.com/api/workspace/jobs/update)", + version: "0.0.1", + type: "action", + props: { + app, + jobId: { + propDefinition: [ + app, + "jobId", + ], + }, + newSettings: { + type: "string", + label: "New Settings", + description: `The updated job settings. JSON string format with only the fields you want to update. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/update#new_settings) + +**Example:** +\`\`\`json +{ + "name": "New Job Name" +} +\`\`\` + `, + optional: true, + }, + fieldsToRemove: { + type: "string[]", + label: "Fields to Remove", + description: "List of field paths to remove from the job settings. [See the API documentation](https://docs.databricks.com/api/workspace/jobs/update#fields_to_remove)", + optional: true, + }, + }, + async run({ $ }) { + const { + app, + jobId, + newSettings, + fieldsToRemove, + } = this; + + await app.updateJob({ + $, + data: { + job_id: jobId, + new_settings: utils.parseJsonInput(newSettings), + fields_to_remove: utils.parseJsonInput(fieldsToRemove), + }, + }); + + $.export("$summary", `Successfully updated job with ID \`${jobId}\`.`); + + return { + success: true, + }; + }, +}; diff --git a/components/databricks/common/constants.mjs b/components/databricks/common/constants.mjs index 4b53df7a27a86..620a90edbd8aa 100644 --- a/components/databricks/common/constants.mjs +++ b/components/databricks/common/constants.mjs @@ -1,3 +1,13 @@ +const DOMAIN_PLACEHOLDER = "{domain}"; +const BASE_URL = `https://${DOMAIN_PLACEHOLDER}.cloud.databricks.com`; + +const VERSION_PATH = { + V2_0: "/api/2.0", + V2_2: "/api/2.2", +}; + +const DEFAULT_LIMIT = 100; + export const CLUSTER_SIZES = [ "2X-Small", "X-Small", @@ -12,4 +22,8 @@ export const CLUSTER_SIZES = [ export default { CLUSTER_SIZES, + DOMAIN_PLACEHOLDER, + BASE_URL, + VERSION_PATH, + DEFAULT_LIMIT, }; diff --git a/components/databricks/common/utils.mjs b/components/databricks/common/utils.mjs index 40bdddb456371..cd0868241d928 100644 --- a/components/databricks/common/utils.mjs +++ b/components/databricks/common/utils.mjs @@ -28,4 +28,9 @@ const parseObject = (obj) => { export default { parseObject, + parseJsonInput: (value) => { + return value + ? parseObject(value) + : undefined; + }, }; diff --git a/components/databricks/databricks.app.mjs b/components/databricks/databricks.app.mjs index 8e8e317268b14..d4f941e5c922c 100644 --- a/components/databricks/databricks.app.mjs +++ b/components/databricks/databricks.app.mjs @@ -1,4 +1,5 @@ import { axios } from "@pipedream/platform"; +import constants from "./common/constants.mjs"; export default { type: "app", @@ -8,14 +9,30 @@ export default { type: "string", label: "Job", description: "Identifier of a job", - async options() { - const { jobs } = await this.listJobs(); - return jobs?.map(({ + async options({ prevContext }) { + if (prevContext.pageToken === null) { + return []; + } + const { + jobs, next_page_token: pageToken, + } = await this.listJobs({ + params: { + page_token: prevContext.pageToken, + limit: constants.DEFAULT_LIMIT, + }, + }); + const options = jobs?.map(({ job_id: value, settings, }) => ({ value, - label: settings.name, + label: settings?.name || value, })) || []; + return { + options, + context: { + pageToken: pageToken || null, + }, + }; }, }, runId: { @@ -82,9 +99,9 @@ export default { }, }, methods: { - - _baseUrl() { - return `https://${this.$auth.domain}.cloud.databricks.com/api/2.0`; + getUrl(path, versionPath = constants.VERSION_PATH.V2_0) { + const baseUrl = constants.BASE_URL.replace(constants.DOMAIN_PLACEHOLDER, this.$auth.domain); + return `${baseUrl}${versionPath}${path}`; }, _headers() { return { @@ -92,38 +109,142 @@ export default { }; }, _makeRequest({ - $ = this, - path, - ...args - }) { + $ = this, path, versionPath, ...args + } = {}) { return axios($, { - url: `${this._baseUrl()}${path}`, + url: this.getUrl(path, versionPath), headers: this._headers(), ...args, }); }, + createJob(args = {}) { + return this._makeRequest({ + path: "/jobs/create", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, listJobs(args = {}) { return this._makeRequest({ path: "/jobs/list", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getJob(args = {}) { + return this._makeRequest({ + path: "/jobs/get", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + resetJob(args = {}) { + return this._makeRequest({ + path: "/jobs/reset", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + updateJob(args = {}) { + return this._makeRequest({ + path: "/jobs/update", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + deleteJob(args = {}) { + return this._makeRequest({ + path: "/jobs/delete", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + runJobNow(args = {}) { + return this._makeRequest({ + path: "/jobs/run-now", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/get", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, listRuns(args = {}) { return this._makeRequest({ path: "/jobs/runs/list", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + cancelRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/cancel", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + cancelAllRuns(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/cancel-all", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, getRunOutput(args = {}) { return this._makeRequest({ path: "/jobs/runs/get-output", + versionPath: constants.VERSION_PATH.V2_2, ...args, }); }, - runJobNow(args = {}) { + deleteRun(args = {}) { return this._makeRequest({ - path: "/jobs/run-now", + path: "/jobs/runs/delete", method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + repairRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/repair", + method: "POST", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + exportRun(args = {}) { + return this._makeRequest({ + path: "/jobs/runs/export", + versionPath: constants.VERSION_PATH.V2_2, + ...args, + }); + }, + getJobPermissions({ + jobId, ...args + }) { + return this._makeRequest({ + path: `/permissions/jobs/${jobId}`, + ...args, + }); + }, + setJobPermissions({ + jobId, ...args + }) { + return this._makeRequest({ + path: `/permissions/jobs/${jobId}`, + method: "PUT", ...args, }); }, @@ -241,7 +362,6 @@ export default { ...args, }); }, - setSQLWarehousePermissions({ warehouseId, ...args }) { @@ -251,5 +371,45 @@ export default { ...args, }); }, + async paginate({ + requestor, requestorArgs = {}, + maxRequests = 3, resultsKey = "jobs", + }) { + const allResults = []; + let requestCount = 0; + let nextPageToken = null; + let hasMore = true; + + while (hasMore && requestCount < maxRequests) { + try { + const response = await requestor({ + ...requestorArgs, + params: { + ...requestorArgs.params, + page_token: nextPageToken, + }, + }); + + requestCount++; + + const results = response[resultsKey] || []; + + allResults.push(...results); + + nextPageToken = response.next_page_token; + hasMore = !!nextPageToken; + + if (results.length === 0) { + hasMore = false; + } + + } catch (error) { + console.error(`Pagination error on request ${requestCount}:`, error); + throw error; + } + } + + return allResults; + }, }, }; diff --git a/components/databricks/package.json b/components/databricks/package.json index d399a1889095f..c82b72576579b 100644 --- a/components/databricks/package.json +++ b/components/databricks/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/databricks", - "version": "0.3.0", + "version": "0.4.0", "description": "Pipedream Databricks Components", "main": "databricks.app.mjs", "keywords": [