diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..f85346d8 Binary files /dev/null and b/.DS_Store differ diff --git a/bundle_config_schema.json b/bundle_config_schema.json new file mode 100644 index 00000000..0db03c39 --- /dev/null +++ b/bundle_config_schema.json @@ -0,0 +1,9846 @@ +{ + "$defs": { + "bool": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "float64": { + "oneOf": [ + { + "type": "number" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.App": { + "oneOf": [ + { + "type": "object", + "properties": { + "active_deployment": { + "description": "The active deployment of the app. A deployment is considered active when it has been deployed\nto the app compute.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment" + }, + "app_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationStatus" + }, + "budget_policy_id": { + "$ref": "#/$defs/string" + }, + "compute_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeStatus" + }, + "config": { + "$ref": "#/$defs/map/interface" + }, + "create_time": { + "description": "The creation time of the app. Formatted timestamp in ISO 6801.", + "$ref": "#/$defs/string" + }, + "creator": { + "description": "The email of the user that created the app.", + "$ref": "#/$defs/string" + }, + "default_source_code_path": { + "description": "The default workspace file system path of the source code from which app deployment are\ncreated. This field tracks the workspace source code path of the last active deployment.", + "$ref": "#/$defs/string" + }, + "description": { + "description": "The description of the app.", + "$ref": "#/$defs/string" + }, + "effective_budget_policy_id": { + "$ref": "#/$defs/string" + }, + "effective_user_api_scopes": { + "description": "The effective api scopes granted to the user access token.", + "$ref": "#/$defs/slice/string" + }, + "id": { + "description": "The unique identifier of the app.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.\nIt must be unique within the workspace.", + "$ref": "#/$defs/string" + }, + "oauth2_app_client_id": { + "$ref": "#/$defs/string" + }, + "oauth2_app_integration_id": { + "$ref": "#/$defs/string" + }, + "pending_deployment": { + "description": "The pending deployment of the app. A deployment is considered pending when it is being prepared\nfor deployment to the app compute.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeployment" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.AppPermission" + }, + "resources": { + "description": "Resources for the app.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/apps.AppResource" + }, + "service_principal_client_id": { + "$ref": "#/$defs/string" + }, + "service_principal_id": { + "$ref": "#/$defs/int64" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "source_code_path": { + "$ref": "#/$defs/string" + }, + "update_time": { + "description": "The update time of the app. Formatted timestamp in ISO 6801.", + "$ref": "#/$defs/string" + }, + "updater": { + "description": "The email of the user that last updated the app.", + "$ref": "#/$defs/string" + }, + "url": { + "description": "The URL of the app once it is deployed.", + "$ref": "#/$defs/string" + }, + "user_api_scopes": { + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "source_code_path", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.AppPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.AppPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.AppPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_USE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Cluster": { + "oneOf": [ + { + "type": "object", + "description": "Contains a snapshot of the latest user specified settings that were used to create/edit the cluster.", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nThree kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\nFor job clusters, the cluster name is automatically set based on the job and job run IDs.", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n\nThis field, along with node_type_id, should not be set if virtual_cluster_size is set.\nIf both driver_node_type_id, node_type_id, and virtual_cluster_size are specified, driver_node_type_id and node_type_id take precedence.", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified.\nThe scripts are executed sequentially in the order provided.\nIf `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "is_single_node": { + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`", + "$ref": "#/$defs/bool" + }, + "kind": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Kind" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.ClusterPermission" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "remote_disk_throughput": { + "description": "If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported for GCP HYPERDISK_BALANCED disks.", + "$ref": "#/$defs/int" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "total_initial_remote_disk_size": { + "description": "If set, what the total initial volume size (in GB) of the remote disks should be. Currently only supported for GCP HYPERDISK_BALANCED disks.", + "$ref": "#/$defs/int" + }, + "use_ml_runtime": { + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.", + "$ref": "#/$defs/bool" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false, + "markdownDescription": "The cluster resource defines an [all-purpose cluster](https://docs.databricks.com/api/workspace/clusters/create)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ClusterPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ClusterPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ClusterPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_RESTART", + "CAN_ATTACH_TO" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Dashboard": { + "oneOf": [ + { + "type": "object", + "properties": { + "create_time": { + "description": "The timestamp of when the dashboard was created.", + "$ref": "#/$defs/string" + }, + "dashboard_id": { + "description": "UUID identifying the dashboard.", + "$ref": "#/$defs/string" + }, + "display_name": { + "description": "The display name of the dashboard.", + "$ref": "#/$defs/string" + }, + "embed_credentials": { + "$ref": "#/$defs/bool" + }, + "etag": { + "description": "The etag for the dashboard. Can be optionally provided on updates to ensure that the dashboard\nhas not been modified since the last read.\nThis field is excluded in List Dashboards responses.", + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle_state": { + "description": "The state of the dashboard resource. Used for tracking trashed status.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/dashboards.LifecycleState" + }, + "parent_path": { + "description": "The workspace path of the folder containing the dashboard. Includes leading slash and no\ntrailing slash.\nThis field is excluded in List Dashboards responses.", + "$ref": "#/$defs/string" + }, + "path": { + "description": "The workspace path of the dashboard asset, including the file name.\nExported dashboards always have the file extension `.lvdash.json`.\nThis field is excluded in List Dashboards responses.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.DashboardPermission" + }, + "serialized_dashboard": { + "description": "The contents of the dashboard in serialized string form.\nThis field is excluded in List Dashboards responses.\nUse the [get dashboard API](https://docs.databricks.com/api/workspace/lakeview/get)\nto retrieve an example response, which includes the `serialized_dashboard` field.\nThis field provides the structure of the JSON string that represents the dashboard's\nlayout and components.", + "$ref": "#/$defs/interface" + }, + "update_time": { + "description": "The timestamp of when the dashboard was last updated by the user.\nThis field is excluded in List Dashboards responses.", + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "description": "The warehouse ID used to run the dashboard.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "markdownDescription": "The dashboard resource allows you to manage [AI/BI dashboards](https://docs.databricks.com/api/workspace/lakeview/create) in a bundle. For information about AI/BI dashboards, see [link](https://docs.databricks.com/dashboards/index.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DashboardPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DashboardPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DashboardPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_READ", + "CAN_RUN", + "CAN_EDIT", + "CAN_MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseCatalog": { + "oneOf": [ + { + "type": "object", + "properties": { + "create_database_if_not_exists": { + "$ref": "#/$defs/bool" + }, + "database_instance_name": { + "$ref": "#/$defs/string" + }, + "database_name": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/string" + }, + "uid": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "database_instance_name", + "database_name", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseInstance": { + "oneOf": [ + { + "type": "object", + "description": "A DatabaseInstance represents a logical Postgres instance, comprised of both compute and storage.", + "properties": { + "capacity": { + "$ref": "#/$defs/string" + }, + "child_instance_refs": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/database.DatabaseInstanceRef" + }, + "creation_time": { + "$ref": "#/$defs/string" + }, + "creator": { + "$ref": "#/$defs/string" + }, + "effective_enable_readable_secondaries": { + "$ref": "#/$defs/bool" + }, + "effective_node_count": { + "$ref": "#/$defs/int" + }, + "effective_retention_window_in_days": { + "$ref": "#/$defs/int" + }, + "effective_stopped": { + "$ref": "#/$defs/bool" + }, + "enable_readable_secondaries": { + "$ref": "#/$defs/bool" + }, + "name": { + "$ref": "#/$defs/string" + }, + "node_count": { + "$ref": "#/$defs/int" + }, + "parent_instance_ref": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.DatabaseInstanceRef" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.DatabaseInstancePermission" + }, + "pg_version": { + "$ref": "#/$defs/string" + }, + "read_only_dns": { + "$ref": "#/$defs/string" + }, + "read_write_dns": { + "$ref": "#/$defs/string" + }, + "retention_window_in_days": { + "$ref": "#/$defs/int" + }, + "state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.DatabaseInstanceState" + }, + "stopped": { + "$ref": "#/$defs/bool" + }, + "uid": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseInstancePermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DatabaseInstancePermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseInstancePermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_CREATE", + "CAN_USE", + "CAN_MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Grant": { + "oneOf": [ + { + "type": "object", + "properties": { + "principal": { + "description": "The name of the principal that will be granted privileges", + "$ref": "#/$defs/string" + }, + "privileges": { + "description": "The privileges to grant to the specified entity", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "privileges", + "principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Job": { + "oneOf": [ + { + "type": "object", + "properties": { + "budget_policy_id": { + "description": "The id of the user specified budget policy to use for this job.\nIf not specified, a default budget policy may be applied when creating or modifying the job.\nSee `effective_budget_policy_id` for the budget policy used by this workload.", + "$ref": "#/$defs/string" + }, + "continuous": { + "description": "An optional continuous property for this job. The continuous property will ensure that there is always one run executing. Only one of `schedule` and `continuous` can be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Continuous" + }, + "description": { + "description": "An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this job begin or complete as well as when this job is deleted.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications" + }, + "environments": { + "description": "A list of task execution environment specifications that can be referenced by serverless tasks of this job.\nAn environment is required to be present for serverless tasks.\nFor serverless notebook tasks, the environment is accessible in the notebook environment panel.\nFor other serverless tasks, the task environment is required to be specified using environment_key in the task settings.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + }, + "git_source": { + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitSource" + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_clusters": { + "description": "A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + }, + "max_concurrent_runs": { + "description": "An optional maximum allowed number of concurrent runs of the job.\nSet this value if you want to be able to execute multiple runs of the same job concurrently.\nThis is useful for example if you trigger your job on a frequent schedule and want to allow consecutive runs to overlap with each other, or if you want to trigger multiple runs which differ by their input parameters.\nThis setting affects only new runs. For example, suppose the job’s concurrency is 4 and there are 4 concurrent active runs. Then setting the concurrency to 3 won’t kill any of the active runs.\nHowever, from then on, new runs are skipped unless there are fewer than 3 active runs.\nThis value cannot exceed 1000. Setting this value to `0` causes all new runs to be skipped.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "An optional name for the job. The maximum length is 4096 bytes in UTF-8 encoding.", + "$ref": "#/$defs/string" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobNotificationSettings" + }, + "parameters": { + "description": "Job-level parameter definitions", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + }, + "performance_target": { + "description": "The performance mode on a serverless job. This field determines the level of compute performance or cost-efficiency for the run.\n\n* `STANDARD`: Enables cost-efficient execution of serverless workloads.\n* `PERFORMANCE_OPTIMIZED`: Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PerformanceTarget" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.JobPermission" + }, + "queue": { + "description": "The queue settings of the job.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.QueueSettings" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs" + }, + "schedule": { + "description": "An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.CronSchedule" + }, + "tags": { + "description": "A map of tags associated with the job. These are forwarded to the cluster as cluster tags for jobs clusters, and are subject to the same limitations as cluster tags. A maximum of 25 tags can be added to the job.", + "$ref": "#/$defs/map/string" + }, + "tasks": { + "description": "A list of task specifications to be executed by this job.\nIt supports up to 1000 elements in write endpoints (:method:jobs/create, :method:jobs/reset, :method:jobs/update, :method:jobs/submit).\nRead endpoints return only 100 tasks. If more than 100 tasks are available, you can paginate through them using :method:jobs/get. Use the `next_page_token` field at the object root to determine if more results are available.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Task" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "trigger": { + "description": "A configuration to trigger a run when certain conditions are met. The default behavior is that the job runs only when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings" + }, + "usage_policy_id": { + "description": "The id of the user specified usage policy to use for this job.\nIf not specified, a default usage policy may be applied when creating or modifying the job.\nSee `effective_usage_policy_id` for the usage policy used by this workload.", + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this job begin or complete.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false, + "markdownDescription": "The job resource allows you to define [jobs and their corresponding tasks](https://docs.databricks.com/api/workspace/jobs/create) in your bundle. For information about jobs, see [link](https://docs.databricks.com/jobs/index.html). For a tutorial that uses a Databricks Asset Bundles template to create a job, see [link](https://docs.databricks.com/dev-tools/bundles/jobs-tutorial.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.JobPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.JobPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.JobPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_MANAGE_RUN", + "CAN_VIEW", + "IS_OWNER" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "oneOf": [ + { + "type": "object", + "description": "An experiment and its metadata.", + "properties": { + "artifact_location": { + "description": "Location where artifacts for the experiment are stored.", + "$ref": "#/$defs/string" + }, + "creation_time": { + "description": "Creation time", + "$ref": "#/$defs/int64" + }, + "experiment_id": { + "description": "Unique identifier for the experiment.", + "$ref": "#/$defs/string" + }, + "last_update_time": { + "description": "Last update time", + "$ref": "#/$defs/int64" + }, + "lifecycle_stage": { + "description": "Current life cycle stage of the experiment: \"active\" or \"deleted\".\nDeleted experiments are not returned by APIs.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "Human readable name that identifies the experiment.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission" + }, + "tags": { + "description": "Tags: Additional metadata key-value pairs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + "additionalProperties": false, + "markdownDescription": "The experiment resource allows you to define [MLflow experiments](https://docs.databricks.com/api/workspace/experiments/createexperiment) in a bundle. For information about MLflow experiments, see [link](https://docs.databricks.com/mlflow/experiments.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperimentPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperimentPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_EDIT", + "CAN_READ" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModel": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "description": "Optional description for registered model.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "Register models under this name", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.MlflowModelPermission" + }, + "tags": { + "description": "Additional metadata for registered model.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + } + }, + "additionalProperties": false, + "required": [ + "name" + ], + "markdownDescription": "The model resource allows you to define [legacy models](https://docs.databricks.com/api/workspace/modelregistry/createmodel) in bundles. Databricks recommends you use Unity Catalog [registered models](https://docs.databricks.com/dev-tools/bundles/reference.html#registered-model) instead." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModelPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowModelPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModelPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_EDIT", + "CAN_MANAGE", + "CAN_MANAGE_STAGING_VERSIONS", + "CAN_MANAGE_PRODUCTION_VERSIONS", + "CAN_READ" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "oneOf": [ + { + "type": "object", + "properties": { + "ai_gateway": { + "description": "The AI Gateway configuration for the serving endpoint. NOTE: External model, provisioned throughput, and pay-per-token endpoints are fully supported; agent endpoints currently only support inference tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayConfig" + }, + "budget_policy_id": { + "description": "The budget policy to be applied to the serving endpoint.", + "$ref": "#/$defs/string" + }, + "config": { + "description": "The core config of the serving endpoint.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointCoreConfigInput" + }, + "description": { + "$ref": "#/$defs/string" + }, + "email_notifications": { + "description": "Email notification settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EmailNotifications" + }, + "name": { + "description": "The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.\nAn endpoint name can consist of alphanumeric characters, dashes, and underscores.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermission" + }, + "rate_limits": { + "description": "Rate limits to be applied to the serving endpoint. NOTE: this field is deprecated, please use AI Gateway to manage rate limits.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.RateLimit", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "route_optimized": { + "description": "Enable route optimization for the serving endpoint.", + "$ref": "#/$defs/bool" + }, + "tags": { + "description": "Tags to be attached to the serving endpoint and automatically propagated to billing logs.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + "additionalProperties": false, + "required": [ + "name" + ], + "markdownDescription": "The model_serving_endpoint resource allows you to define [model serving endpoints](https://docs.databricks.com/api/workspace/servingendpoints/create). See [link](https://docs.databricks.com/machine-learning/model-serving/manage-serving-endpoints.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpointPermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpointPermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_QUERY", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Permission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "description": "The name of the group that has the permission set in level.", + "$ref": "#/$defs/string" + }, + "level": { + "description": "The allowed permission for user, group, service principal defined for this permission.", + "$ref": "#/$defs/string" + }, + "service_principal_name": { + "description": "The name of the service principal that has the permission set in level.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The name of the user that has the permission set in level.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Pipeline": { + "oneOf": [ + { + "type": "object", + "properties": { + "allow_duplicate_names": { + "description": "If false, deployment will fail if name conflicts with that of another pipeline.", + "$ref": "#/$defs/bool" + }, + "budget_policy_id": { + "description": "Budget policy of this pipeline.", + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "catalog": { + "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", + "$ref": "#/$defs/string" + }, + "channel": { + "description": "DLT Release Channel that specifies which version to use.", + "$ref": "#/$defs/string" + }, + "clusters": { + "description": "Cluster settings for this pipeline deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + }, + "configuration": { + "description": "String-String configuration for this pipeline execution.", + "$ref": "#/$defs/map/string" + }, + "continuous": { + "description": "Whether the pipeline is continuous or triggered. This replaces `trigger`.", + "$ref": "#/$defs/bool" + }, + "development": { + "description": "Whether the pipeline is in Development mode. Defaults to false.", + "$ref": "#/$defs/bool" + }, + "edition": { + "description": "Pipeline product edition.", + "$ref": "#/$defs/string" + }, + "environment": { + "description": "Environment specification for this pipeline used to install dependencies.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelinesEnvironment" + }, + "event_log": { + "description": "Event log configuration for this pipeline", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.EventLogSpec" + }, + "filters": { + "description": "Filters on which Pipeline packages to include in the deployed graph.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Filters" + }, + "gateway_definition": { + "description": "The definition of a gateway pipeline to support change data capture.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "id": { + "description": "Unique identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "ingestion_definition": { + "description": "The configuration for a managed ingestion pipeline. These settings cannot be used with the 'libraries', 'schema', 'target', or 'catalog' settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinition" + }, + "libraries": { + "description": "Libraries or code needed by this deployment.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + }, + "name": { + "description": "Friendly identifier for this pipeline.", + "$ref": "#/$defs/string" + }, + "notifications": { + "description": "List of notification settings for this pipeline.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.PipelinePermission" + }, + "photon": { + "description": "Whether Photon is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "restart_window": { + "description": "Restart window of this pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RestartWindow", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "root_path": { + "description": "Root path for this pipeline.\nThis is used as the root directory when editing the pipeline in the Databricks user interface and it is\nadded to sys.path when executing Python sources during pipeline execution.", + "$ref": "#/$defs/string" + }, + "run_as": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.RunAs", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "schema": { + "description": "The default schema (database) where tables are read from or published to.", + "$ref": "#/$defs/string" + }, + "serverless": { + "description": "Whether serverless compute is enabled for this pipeline.", + "$ref": "#/$defs/bool" + }, + "storage": { + "description": "DBFS root directory for storing checkpoints and tables.", + "$ref": "#/$defs/string" + }, + "tags": { + "description": "A map of tags associated with the pipeline.\nThese are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations.\nA maximum of 25 tags can be added to the pipeline.", + "$ref": "#/$defs/map/string" + }, + "target": { + "description": "Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.", + "$ref": "#/$defs/string" + }, + "trigger": { + "description": "Which pipeline trigger to use. Deprecated: Use `continuous` instead.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger", + "deprecationMessage": "Use continuous instead", + "deprecated": true + } + }, + "additionalProperties": false, + "markdownDescription": "The pipeline resource allows you to create Delta Live Tables [pipelines](https://docs.databricks.com/api/workspace/pipelines/create). For information about pipelines, see [link](https://docs.databricks.com/dlt/index.html). For a tutorial that uses the Databricks Asset Bundles template to create a pipeline, see [link](https://docs.databricks.com/dev-tools/bundles/pipelines-tutorial.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelinePermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelinePermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelinePermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "IS_OWNER", + "CAN_RUN", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.QualityMonitor": { + "oneOf": [ + { + "type": "object", + "properties": { + "assets_dir": { + "description": "[Create:REQ Update:IGN] Field for specifying the absolute path to a custom directory to store data-monitoring\nassets. Normally prepopulated to a default user location via UI and Python APIs.", + "$ref": "#/$defs/string" + }, + "baseline_table_name": { + "description": "[Create:OPT Update:OPT] Baseline table name.\nBaseline data is used to compute drift from the data in the monitored `table_name`.\nThe baseline table and the monitored table shall have the same schema.", + "$ref": "#/$defs/string" + }, + "custom_metrics": { + "description": "[Create:OPT Update:OPT] Custom metrics.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + }, + "data_classification_config": { + "description": "[Create:OPT Update:OPT] Data classification related config.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDataClassificationConfig", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "inference_log": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLog" + }, + "latest_monitor_failure_msg": { + "description": "[Create:ERR Update:IGN] The latest error message for a monitor failure.", + "$ref": "#/$defs/string" + }, + "notifications": { + "description": "[Create:OPT Update:OPT] Field for specifying notification settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorNotifications" + }, + "output_schema_name": { + "description": "[Create:REQ Update:REQ] Schema where output tables are created. Needs to be in 2-level format {catalog}.{schema}", + "$ref": "#/$defs/string" + }, + "schedule": { + "description": "[Create:OPT Update:OPT] The monitor schedule.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedule" + }, + "skip_builtin_dashboard": { + "description": "Whether to skip creating a default dashboard summarizing data quality metrics.", + "$ref": "#/$defs/bool" + }, + "slicing_exprs": { + "description": "[Create:OPT Update:OPT] List of column expressions to slice data with for targeted analysis. The data is grouped by\neach expression independently, resulting in a separate slice for each predicate and its\ncomplements. For example `slicing_exprs=[“col_1”, “col_2 \u003e 10”]` will generate the following\nslices: two slices for `col_2 \u003e 10` (True and False), and one slice per unique value in\n`col1`. For high-cardinality columns, only the top 100 unique values by frequency will\ngenerate slices.", + "$ref": "#/$defs/slice/string" + }, + "snapshot": { + "description": "Configuration for monitoring snapshot tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorSnapshot" + }, + "table_name": { + "$ref": "#/$defs/string" + }, + "time_series": { + "description": "Configuration for monitoring time series tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorTimeSeries" + }, + "warehouse_id": { + "description": "Optional argument to specify the warehouse for dashboard creation. If not specified, the first running\nwarehouse will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "table_name", + "assets_dir", + "output_schema_name" + ], + "markdownDescription": "The quality_monitor resource allows you to define a Unity Catalog [table monitor](https://docs.databricks.com/api/workspace/qualitymonitors/create). For information about monitors, see [link](https://docs.databricks.com/machine-learning/model-serving/monitor-diagnose-endpoints.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.RegisteredModel": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the registered model reside", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "The comment attached to the registered model", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "The name of the registered model", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the registered model resides", + "$ref": "#/$defs/string" + }, + "storage_location": { + "description": "The storage location on the cloud under which model version data files are stored", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name", + "schema_name" + ], + "markdownDescription": "The registered model resource allows you to define models in Unity Catalog. For information about Unity Catalog [registered models](https://docs.databricks.com/api/workspace/registeredmodels/create), see [link](https://docs.databricks.com/machine-learning/manage-model-lifecycle/index.html)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "Name of parent catalog.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description.", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Grant" + }, + "name": { + "description": "Name of schema, relative to parent catalog.", + "$ref": "#/$defs/string" + }, + "properties": { + "$ref": "#/$defs/map/string" + }, + "storage_root": { + "description": "Storage root URL for managed tables within schema.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name" + ], + "markdownDescription": "The schema resource type allows you to define Unity Catalog [schemas](https://docs.databricks.com/api/workspace/schemas/create) for tables and other assets in your workflows and pipelines created as part of a bundle. A schema, different from other resource types, has the following limitations:\n\n- The owner of a schema resource is always the deployment user, and cannot be changed. If `run_as` is specified in the bundle, it will be ignored by operations on the schema.\n- Only fields supported by the corresponding [Schemas object create API](https://docs.databricks.com/api/workspace/schemas/create) are available for the schema resource. For example, `enable_predictive_optimization` is not supported as it is only available on the [update API](https://docs.databricks.com/api/workspace/schemas/update)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScope": { + "oneOf": [ + { + "type": "object", + "properties": { + "backend_type": { + "description": "The backend type the scope will be created with. If not specified, will default to `DATABRICKS`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/workspace.ScopeBackendType" + }, + "keyvault_metadata": { + "description": "The metadata for the secret scope if the `backend_type` is `AZURE_KEYVAULT`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/workspace.AzureKeyVaultSecretScopeMetadata" + }, + "name": { + "description": "Scope name requested by the user. Scope names are unique.", + "$ref": "#/$defs/string" + }, + "permissions": { + "description": "The permissions to apply to the secret scope. Permissions are managed via secret scope ACLs.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.SecretScopePermission" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScopePermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "description": "The name of the group that has the permission set in level. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + }, + "level": { + "description": "The allowed permission for user, group, service principal defined for this permission.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScopePermissionLevel" + }, + "service_principal_name": { + "description": "The application ID of an active service principal. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The name of the user that has the permission set in level. This field translates to a `principal` field in secret scope ACL.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScopePermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "READ", + "WRITE", + "MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SqlWarehouse": { + "oneOf": [ + { + "type": "object", + "properties": { + "auto_stop_mins": { + "description": "The amount of time in minutes that a SQL warehouse must be idle (i.e., no RUNNING queries) before\nit is automatically stopped.\n\nSupported values:\n - Must be \u003e= 0 mins for serverless warehouses\n - Must be == 0 or \u003e= 10 mins for non-serverless warehouses\n - 0 indicates no autostop.\n\nDefaults to 120 mins", + "$ref": "#/$defs/int" + }, + "channel": { + "description": "Channel Details", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.Channel" + }, + "cluster_size": { + "description": "Size of the clusters allocated for this warehouse.\nIncreasing the size of a spark cluster allows you to run larger queries on it.\nIf you want to increase the number of concurrent queries, please tune max_num_clusters.\n\nSupported values:\n - 2X-Small\n - X-Small\n - Small\n - Medium\n - Large\n - X-Large\n - 2X-Large\n - 3X-Large\n - 4X-Large\n", + "$ref": "#/$defs/string" + }, + "creator_name": { + "description": "warehouse creator name", + "$ref": "#/$defs/string" + }, + "enable_photon": { + "description": "Configures whether the warehouse should use Photon optimized clusters.\n\nDefaults to true.", + "$ref": "#/$defs/bool" + }, + "enable_serverless_compute": { + "description": "Configures whether the warehouse should use serverless compute", + "$ref": "#/$defs/bool" + }, + "instance_profile_arn": { + "description": "Deprecated. Instance profile used to pass IAM role to the cluster", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "max_num_clusters": { + "description": "Maximum number of clusters that the autoscaler will create to handle concurrent queries.\n\nSupported values:\n - Must be \u003e= min_num_clusters\n - Must be \u003c= 30.\n\nDefaults to min_clusters if unset.", + "$ref": "#/$defs/int" + }, + "min_num_clusters": { + "description": "Minimum number of available clusters that will be maintained for this SQL warehouse.\nIncreasing this will ensure that a larger number of clusters are always running and therefore may reduce\nthe cold start time for new queries. This is similar to reserved vs. revocable cores in a resource manager.\n\nSupported values:\n - Must be \u003e 0\n - Must be \u003c= min(max_num_clusters, 30)\n\nDefaults to 1", + "$ref": "#/$defs/int" + }, + "name": { + "description": "Logical name for the cluster.\n\nSupported values:\n - Must be unique within an org.\n - Must be less than 100 characters.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.SqlWarehousePermission" + }, + "spot_instance_policy": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.SpotInstancePolicy" + }, + "tags": { + "description": "A set of key-value pairs that will be tagged on all resources (e.g., AWS instances and EBS volumes) associated\nwith this SQL warehouse.\n\nSupported values:\n - Number of tags \u003c 45.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.EndpointTags" + }, + "warehouse_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.CreateWarehouseRequestWarehouseType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SqlWarehousePermission": { + "oneOf": [ + { + "type": "object", + "properties": { + "group_name": { + "$ref": "#/$defs/string" + }, + "level": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SqlWarehousePermissionLevel" + }, + "service_principal_name": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "level" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SqlWarehousePermissionLevel": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_USE", + "CAN_MONITOR", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SyncedDatabaseTable": { + "oneOf": [ + { + "type": "object", + "properties": { + "data_synchronization_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableStatus" + }, + "database_instance_name": { + "$ref": "#/$defs/string" + }, + "effective_database_instance_name": { + "$ref": "#/$defs/string" + }, + "effective_logical_database_name": { + "$ref": "#/$defs/string" + }, + "logical_database_name": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/string" + }, + "spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableSpec" + }, + "unity_catalog_provisioning_state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.ProvisioningInfoState" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Volume": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the volume are", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "The comment attached to the volume", + "$ref": "#/$defs/string" + }, + "grants": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.VolumeGrant" + }, + "name": { + "description": "The name of the volume", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the volume is", + "$ref": "#/$defs/string" + }, + "storage_location": { + "description": "The storage location on the cloud", + "$ref": "#/$defs/string" + }, + "volume_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.VolumeType" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "name", + "schema_name" + ], + "markdownDescription": "The volume resource type allows you to define and create Unity Catalog [volumes](https://docs.databricks.com/api/workspace/volumes/create) as part of a bundle. When deploying a bundle with a volume defined, note that:\n\n- A volume cannot be referenced in the `artifact_path` for the bundle until it exists in the workspace. Hence, if you want to use Databricks Asset Bundles to create the volume, you must first define the volume in the bundle, deploy it to create the volume, then reference it in the `artifact_path` in subsequent deployments.\n\n- Volumes in the bundle are not prepended with the `dev_${workspace.current_user.short_name}` prefix when the deployment target has `mode: development` configured. However, you can manually configure this prefix. See [custom-presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#custom-presets)." + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.VolumeGrant": { + "oneOf": [ + { + "type": "object", + "properties": { + "principal": { + "$ref": "#/$defs/string" + }, + "privileges": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.VolumeGrantPrivilege" + } + }, + "additionalProperties": false, + "required": [ + "privileges", + "principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.VolumeGrantPrivilege": { + "oneOf": [ + { + "type": "string", + "enum": [ + "ALL_PRIVILEGES", + "APPLY_TAG", + "MANAGE", + "READ_VOLUME", + "WRITE_VOLUME" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.Lookup": { + "oneOf": [ + { + "type": "object", + "properties": { + "alert": { + "description": "The name of the alert for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "cluster": { + "description": "The name of the cluster for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "cluster_policy": { + "description": "The name of the cluster_policy for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "dashboard": { + "description": "The name of the dashboard for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "instance_pool": { + "description": "The name of the instance_pool for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "job": { + "description": "The name of the job for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "metastore": { + "description": "The name of the metastore for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "notification_destination": { + "description": "The name of the notification_destination for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "pipeline": { + "description": "The name of the pipeline for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "query": { + "description": "The name of the query for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "service_principal": { + "description": "The name of the service_principal for which to retrieve an ID.", + "$ref": "#/$defs/string" + }, + "warehouse": { + "description": "The name of the warehouse for which to retrieve an ID.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.TargetVariable": { + "anyOf": [ + { + "type": "object", + "properties": { + "default": { + "description": "The default value for the variable.", + "$ref": "#/$defs/interface" + }, + "description": { + "description": "The description of the variable.", + "$ref": "#/$defs/string" + }, + "lookup": { + "description": "The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup" + }, + "type": { + "description": "The type of the variable.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false + }, + {} + ] + }, + "variable.Variable": { + "type": "object", + "description": "Defines a custom variable for the bundle.", + "properties": { + "default": { + "description": "The default value for the variable.", + "$ref": "#/$defs/interface" + }, + "description": { + "description": "The description of the variable", + "$ref": "#/$defs/string" + }, + "lookup": { + "description": "The name of the alert, cluster_policy, cluster, dashboard, instance_pool, job, metastore, pipeline, query, service_principal, or warehouse object for which to retrieve an ID.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Lookup", + "markdownDescription": "The name of the `alert`, `cluster_policy`, `cluster`, `dashboard`, `instance_pool`, `job`, `metastore`, `pipeline`, `query`, `service_principal`, or `warehouse` object for which to retrieve an ID." + }, + "type": { + "description": "The type of the variable.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.VariableType" + } + }, + "additionalProperties": false, + "markdownDescription": "Defines a custom variable for the bundle. See [variables](https://docs.databricks.com/dev-tools/bundles/settings.html#variables)." + }, + "variable.VariableType": { + "type": "string" + } + }, + "config.Artifact": { + "oneOf": [ + { + "type": "object", + "properties": { + "build": { + "description": "An optional set of build commands to run locally before deployment.", + "$ref": "#/$defs/string" + }, + "dynamic_version": { + "description": "Whether to patch the wheel version dynamically based on the timestamp of the whl file. If this is set to `true`, new code can be deployed without having to update the version in `setup.py` or `pyproject.toml`. This setting is only valid when `type` is set to `whl`. See [\\_](/dev-tools/bundles/settings.md#bundle-syntax-mappings-artifacts).", + "$ref": "#/$defs/bool" + }, + "executable": { + "description": "The executable type. Valid values are `bash`, `sh`, and `cmd`.", + "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" + }, + "files": { + "description": "The relative or absolute path to the built artifact files.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config.ArtifactFile" + }, + "path": { + "description": "The local path of the directory for the artifact.", + "$ref": "#/$defs/string" + }, + "type": { + "description": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactType", + "markdownDescription": "Required if the artifact is a Python wheel. The type of the artifact. Valid values are `whl` and `jar`." + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.ArtifactFile": { + "oneOf": [ + { + "type": "object", + "properties": { + "source": { + "description": "Required. The artifact source file.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "source" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.ArtifactType": { + "type": "string" + }, + "config.Bundle": { + "oneOf": [ + { + "type": "object", + "properties": { + "cluster_id": { + "description": "The ID of a cluster to use to run the bundle.", + "$ref": "#/$defs/string", + "markdownDescription": "The ID of a cluster to use to run the bundle. See [cluster_id](https://docs.databricks.com/dev-tools/bundles/settings.html#cluster_id)." + }, + "compute_id": { + "description": "Deprecated. The ID of the compute to use to run the bundle.", + "$ref": "#/$defs/string" + }, + "databricks_cli_version": { + "description": "The Databricks CLI version to use for the bundle.", + "$ref": "#/$defs/string", + "markdownDescription": "The Databricks CLI version to use for the bundle. See [databricks_cli_version](https://docs.databricks.com/dev-tools/bundles/settings.html#databricks_cli_version)." + }, + "deployment": { + "description": "The definition of the bundle deployment", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Deployment", + "markdownDescription": "The definition of the bundle deployment. For supported attributes see [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." + }, + "git": { + "description": "The Git version control details that are associated with your bundle.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git", + "markdownDescription": "The Git version control details that are associated with your bundle. For supported attributes see [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." + }, + "name": { + "description": "The name of the bundle.", + "$ref": "#/$defs/string" + }, + "uuid": { + "description": "Reserved. A Universally Unique Identifier (UUID) for the bundle that uniquely identifies the bundle in internal Databricks systems. This is generated when a bundle project is initialized using a Databricks template (using the `databricks bundle init` command).", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Command": { + "type": "string" + }, + "config.Deployment": { + "oneOf": [ + { + "type": "object", + "properties": { + "fail_on_active_runs": { + "description": "Whether to fail on active runs. If this is set to true a deployment that is running can be interrupted.", + "$ref": "#/$defs/bool" + }, + "lock": { + "description": "The deployment lock attributes.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Lock" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Experimental": { + "oneOf": [ + { + "type": "object", + "properties": { + "pydabs": { + "description": "The PyDABs configuration.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.PyDABs", + "deprecationMessage": "Deprecated: please use python instead", + "deprecated": true + }, + "python": { + "description": "Configures loading of Python code defined with 'databricks-bundles' package.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Python" + }, + "python_wheel_wrapper": { + "description": "Whether to use a Python wheel wrapper.", + "$ref": "#/$defs/bool" + }, + "scripts": { + "description": "The commands to run.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Command" + }, + "skip_artifact_cleanup": { + "description": "Determines whether to skip cleaning up the .internal folder", + "$ref": "#/$defs/bool" + }, + "skip_name_prefix_for_schema": { + "description": "Skip adding the prefix that is either set in `presets.name_prefix` or computed when `mode: development`\nis set, to the names of UC schemas defined in the bundle.", + "$ref": "#/$defs/bool" + }, + "use_legacy_run_as": { + "description": "Whether to use the legacy run_as behavior.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Git": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "description": "The Git branch name.", + "$ref": "#/$defs/string", + "markdownDescription": "The Git branch name. See [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." + }, + "origin_url": { + "description": "The origin URL of the repository.", + "$ref": "#/$defs/string", + "markdownDescription": "The origin URL of the repository. See [git](https://docs.databricks.com/dev-tools/bundles/settings.html#git)." + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Lock": { + "oneOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether this lock is enabled.", + "$ref": "#/$defs/bool" + }, + "force": { + "description": "Whether to force this lock if it is enabled.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Mode": { + "type": "string" + }, + "config.Presets": { + "oneOf": [ + { + "type": "object", + "properties": { + "artifacts_dynamic_version": { + "description": "Whether to enable dynamic_version on all artifacts.", + "$ref": "#/$defs/bool" + }, + "jobs_max_concurrent_runs": { + "description": "The maximum concurrent runs for a job.", + "$ref": "#/$defs/int" + }, + "name_prefix": { + "description": "The prefix for job runs of the bundle.", + "$ref": "#/$defs/string" + }, + "pipelines_development": { + "description": "Whether pipeline deployments should be locked in development mode.", + "$ref": "#/$defs/bool" + }, + "source_linked_deployment": { + "description": "Whether to link the deployment to the bundle source.", + "$ref": "#/$defs/bool" + }, + "tags": { + "description": "The tags for the bundle deployment.", + "$ref": "#/$defs/map/string" + }, + "trigger_pause_status": { + "description": "A pause status to apply to all job triggers and schedules. Valid values are PAUSED or UNPAUSED.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.PyDABs": { + "oneOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether or not PyDABs (Private Preview) is enabled", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Python": { + "oneOf": [ + { + "type": "object", + "properties": { + "mutators": { + "description": "Mutators contains a list of fully qualified function paths to mutator functions.\n\nExample: [\"my_project.mutators:add_default_cluster\"]", + "$ref": "#/$defs/slice/string" + }, + "resources": { + "description": "Resources contains a list of fully qualified function paths to load resources\ndefined in Python code.\n\nExample: [\"my_project.resources:load_resources\"]", + "$ref": "#/$defs/slice/string" + }, + "venv_path": { + "description": "VEnvPath is path to the virtual environment.\n\nIf enabled, Python code will execute within this environment. If disabled,\nit defaults to using the Python interpreter available in the current shell.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Resources": { + "oneOf": [ + { + "type": "object", + "properties": { + "apps": { + "description": "The app resource defines a Databricks app.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.App", + "markdownDescription": "The app resource defines a [Databricks app](https://docs.databricks.com/api/workspace/apps/create). For information about Databricks Apps, see [link](https://docs.databricks.com/dev-tools/databricks-apps/index.html)." + }, + "clusters": { + "description": "The cluster definitions for the bundle, where each key is the name of a cluster.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Cluster", + "markdownDescription": "The cluster definitions for the bundle, where each key is the name of a cluster. See [clusters](https://docs.databricks.com/dev-tools/bundles/resources.html#clusters)." + }, + "dashboards": { + "description": "The dashboard definitions for the bundle, where each key is the name of the dashboard.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Dashboard", + "markdownDescription": "The dashboard definitions for the bundle, where each key is the name of the dashboard. See [dashboards](https://docs.databricks.com/dev-tools/bundles/resources.html#dashboards)." + }, + "database_catalogs": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.DatabaseCatalog" + }, + "database_instances": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.DatabaseInstance" + }, + "experiments": { + "description": "The experiment definitions for the bundle, where each key is the name of the experiment.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowExperiment", + "markdownDescription": "The experiment definitions for the bundle, where each key is the name of the experiment. See [experiments](https://docs.databricks.com/dev-tools/bundles/resources.html#experiments)." + }, + "jobs": { + "description": "The job definitions for the bundle, where each key is the name of the job.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", + "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)." + }, + "model_serving_endpoints": { + "description": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint", + "markdownDescription": "The model serving endpoint definitions for the bundle, where each key is the name of the model serving endpoint. See [model_serving_endpoints](https://docs.databricks.com/dev-tools/bundles/resources.html#model_serving_endpoints)." + }, + "models": { + "description": "The model definitions for the bundle, where each key is the name of the model.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MlflowModel", + "markdownDescription": "The model definitions for the bundle, where each key is the name of the model. See [models](https://docs.databricks.com/dev-tools/bundles/resources.html#models)." + }, + "pipelines": { + "description": "The pipeline definitions for the bundle, where each key is the name of the pipeline.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Pipeline", + "markdownDescription": "The pipeline definitions for the bundle, where each key is the name of the pipeline. See [pipelines](https://docs.databricks.com/dev-tools/bundles/resources.html#pipelines)." + }, + "quality_monitors": { + "description": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.QualityMonitor", + "markdownDescription": "The quality monitor definitions for the bundle, where each key is the name of the quality monitor. See [quality_monitors](https://docs.databricks.com/dev-tools/bundles/resources.html#quality_monitors)." + }, + "registered_models": { + "description": "The registered model definitions for the bundle, where each key is the name of the Unity Catalog registered model.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.RegisteredModel", + "markdownDescription": "The registered model definitions for the bundle, where each key is the name of the Unity Catalog registered model. See [registered_models](https://docs.databricks.com/dev-tools/bundles/resources.html#registered_models)" + }, + "schemas": { + "description": "The schema definitions for the bundle, where each key is the name of the schema.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Schema", + "markdownDescription": "The schema definitions for the bundle, where each key is the name of the schema. See [schemas](https://docs.databricks.com/dev-tools/bundles/resources.html#schemas)." + }, + "secret_scopes": { + "description": "The secret scope definitions for the bundle, where each key is the name of the secret scope.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SecretScope", + "markdownDescription": "The secret scope definitions for the bundle, where each key is the name of the secret scope. See [secret_scopes](https://docs.databricks.com/dev-tools/bundles/resources.html#secret_scopes)." + }, + "sql_warehouses": { + "description": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SqlWarehouse", + "markdownDescription": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. See [sql_warehouses](https://docs.databricks.com/dev-tools/bundles/resources.html#sql_warehouses)." + }, + "synced_database_tables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" + }, + "volumes": { + "description": "The volume definitions for the bundle, where each key is the name of the volume.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", + "markdownDescription": "The volume definitions for the bundle, where each key is the name of the volume. See [volumes](https://docs.databricks.com/dev-tools/bundles/resources.html#volumes)." + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Script": { + "oneOf": [ + { + "type": "object", + "properties": { + "content": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "content" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Sync": { + "oneOf": [ + { + "type": "object", + "properties": { + "exclude": { + "description": "A list of files or folders to exclude from the bundle.", + "$ref": "#/$defs/slice/string" + }, + "include": { + "description": "A list of files or folders to include in the bundle.", + "$ref": "#/$defs/slice/string" + }, + "paths": { + "description": "The local folder paths, which can be outside the bundle root, to synchronize to the workspace when the bundle is deployed.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Target": { + "oneOf": [ + { + "type": "object", + "properties": { + "artifacts": { + "description": "The artifacts to include in the target deployment.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" + }, + "bundle": { + "description": "The bundle attributes when deploying to this target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" + }, + "cluster_id": { + "description": "The ID of the cluster to use for this target.", + "$ref": "#/$defs/string" + }, + "compute_id": { + "description": "Deprecated. The ID of the compute to use for this target.", + "$ref": "#/$defs/string", + "deprecationMessage": "Deprecated: please use cluster_id instead", + "deprecated": true + }, + "default": { + "description": "Whether this target is the default target.", + "$ref": "#/$defs/bool" + }, + "git": { + "description": "The Git version control settings for the target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Git" + }, + "mode": { + "description": "The deployment mode for the target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Mode", + "markdownDescription": "The deployment mode for the target. Valid values are `development` or `production`. See [link](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html)." + }, + "permissions": { + "description": "The permissions for deploying and running the bundle in the target.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "presets": { + "description": "The deployment presets for the target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets" + }, + "resources": { + "description": "The resource definitions for the target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources" + }, + "run_as": { + "description": "The identity to use to run the bundle.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", + "markdownDescription": "The identity to use to run the bundle, see [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." + }, + "sync": { + "description": "The local paths to sync to the target workspace when a bundle is run or deployed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync" + }, + "variables": { + "description": "The custom variable definitions for the target.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.TargetVariable" + }, + "workspace": { + "description": "The Databricks workspace for the target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Workspace": { + "oneOf": [ + { + "type": "object", + "properties": { + "artifact_path": { + "description": "The artifact path to use within the workspace for both deployments and workflow runs", + "$ref": "#/$defs/string" + }, + "auth_type": { + "description": "The authentication type.", + "$ref": "#/$defs/string" + }, + "azure_client_id": { + "description": "The Azure client ID", + "$ref": "#/$defs/string" + }, + "azure_environment": { + "description": "The Azure environment", + "$ref": "#/$defs/string" + }, + "azure_login_app_id": { + "description": "The Azure login app ID", + "$ref": "#/$defs/string" + }, + "azure_tenant_id": { + "description": "The Azure tenant ID", + "$ref": "#/$defs/string" + }, + "azure_use_msi": { + "description": "Whether to use MSI for Azure", + "$ref": "#/$defs/bool" + }, + "azure_workspace_resource_id": { + "description": "The Azure workspace resource ID", + "$ref": "#/$defs/string" + }, + "client_id": { + "description": "The client ID for the workspace", + "$ref": "#/$defs/string" + }, + "file_path": { + "description": "The file path to use within the workspace for both deployments and workflow runs", + "$ref": "#/$defs/string" + }, + "google_service_account": { + "description": "The Google service account name", + "$ref": "#/$defs/string" + }, + "host": { + "description": "The Databricks workspace host URL", + "$ref": "#/$defs/string" + }, + "profile": { + "description": "The Databricks workspace profile name", + "$ref": "#/$defs/string" + }, + "resource_path": { + "description": "The workspace resource path", + "$ref": "#/$defs/string" + }, + "root_path": { + "description": "The Databricks workspace root path", + "$ref": "#/$defs/string" + }, + "state_path": { + "description": "The workspace state path", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "libs": { + "exec.ExecutableType": { + "type": "string" + } + } + }, + "databricks-sdk-go": { + "service": { + "apps.AppDeployment": { + "oneOf": [ + { + "type": "object", + "properties": { + "create_time": { + "$ref": "#/$defs/string" + }, + "creator": { + "$ref": "#/$defs/string" + }, + "deployment_artifacts": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentArtifacts" + }, + "deployment_id": { + "$ref": "#/$defs/string" + }, + "mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentMode" + }, + "source_code_path": { + "$ref": "#/$defs/string" + }, + "status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus" + }, + "update_time": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppDeploymentArtifacts": { + "oneOf": [ + { + "type": "object", + "properties": { + "source_code_path": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppDeploymentMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "SNAPSHOT", + "AUTO_SYNC" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppDeploymentState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "SUCCEEDED", + "FAILED", + "IN_PROGRESS", + "CANCELLED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppDeploymentStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "message": { + "$ref": "#/$defs/string" + }, + "state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentState" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResource": { + "oneOf": [ + { + "type": "object", + "properties": { + "database": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceDatabase" + }, + "description": { + "description": "Description of the App Resource.", + "$ref": "#/$defs/string" + }, + "job": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob" + }, + "name": { + "description": "Name of the App Resource.", + "$ref": "#/$defs/string" + }, + "secret": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecret" + }, + "serving_endpoint": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpoint" + }, + "sql_warehouse": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouse" + }, + "uc_securable": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceUcSecurable" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceDatabase": { + "oneOf": [ + { + "type": "object", + "properties": { + "database_name": { + "$ref": "#/$defs/string" + }, + "instance_name": { + "$ref": "#/$defs/string" + }, + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceDatabaseDatabasePermission" + } + }, + "additionalProperties": false, + "required": [ + "database_name", + "instance_name", + "permission" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceDatabaseDatabasePermission": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_CONNECT_AND_CREATE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceJob": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/string" + }, + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceJobJobPermission" + } + }, + "additionalProperties": false, + "required": [ + "id", + "permission" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceJobJobPermission": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "IS_OWNER", + "CAN_MANAGE_RUN", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceSecret": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "$ref": "#/$defs/string" + }, + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSecretSecretPermission" + }, + "scope": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "key", + "permission", + "scope" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceSecretSecretPermission": { + "oneOf": [ + { + "type": "string", + "description": "Permission to grant on the secret scope. Supported permissions are: \"READ\", \"WRITE\", \"MANAGE\".", + "enum": [ + "READ", + "WRITE", + "MANAGE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceServingEndpoint": { + "oneOf": [ + { + "type": "object", + "properties": { + "name": { + "$ref": "#/$defs/string" + }, + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceServingEndpointServingEndpointPermission" + } + }, + "additionalProperties": false, + "required": [ + "name", + "permission" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceServingEndpointServingEndpointPermission": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_QUERY", + "CAN_VIEW" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceSqlWarehouse": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/string" + }, + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceSqlWarehouseSqlWarehousePermission" + } + }, + "additionalProperties": false, + "required": [ + "id", + "permission" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceSqlWarehouseSqlWarehousePermission": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CAN_MANAGE", + "CAN_USE", + "IS_OWNER" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceUcSecurable": { + "oneOf": [ + { + "type": "object", + "properties": { + "permission": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceUcSecurableUcSecurablePermission" + }, + "securable_full_name": { + "$ref": "#/$defs/string" + }, + "securable_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResourceUcSecurableUcSecurableType" + } + }, + "additionalProperties": false, + "required": [ + "permission", + "securable_full_name", + "securable_type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceUcSecurableUcSecurablePermission": { + "oneOf": [ + { + "type": "string", + "enum": [ + "READ_VOLUME", + "WRITE_VOLUME" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.AppResourceUcSecurableUcSecurableType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "VOLUME" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.ApplicationState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "DEPLOYING", + "RUNNING", + "CRASHED", + "UNAVAILABLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.ApplicationStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "message": { + "$ref": "#/$defs/string" + }, + "state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ApplicationState" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.ComputeState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "ERROR", + "DELETING", + "STARTING", + "STOPPING", + "UPDATING", + "STOPPED", + "ACTIVE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "apps.ComputeStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "message": { + "$ref": "#/$defs/string" + }, + "state": { + "description": "State of the app compute.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.ComputeState" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorCronSchedule": { + "oneOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Read only field that indicates whether a schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorCronSchedulePauseStatus" + }, + "quartz_cron_expression": { + "description": "The expression that determines when to run the monitor. See [examples](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html).", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "The timezone id (e.g., ``PST``) in which to evaluate the quartz expression.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorCronSchedulePauseStatus": { + "oneOf": [ + { + "type": "string", + "description": "Source link: https://src.dev.databricks.com/databricks/universe/-/blob/elastic-spark-common/api/messages/schedule.proto\nMonitoring workflow schedule pause status.", + "enum": [ + "UNSPECIFIED", + "UNPAUSED", + "PAUSED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorDataClassificationConfig": { + "oneOf": [ + { + "type": "object", + "description": "Data classification related configuration.", + "properties": { + "enabled": { + "description": "Whether to enable data classification.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorDestination": { + "oneOf": [ + { + "type": "object", + "properties": { + "email_addresses": { + "description": "The list of email addresses to send the notification to. A maximum of 5 email addresses is supported.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorInferenceLog": { + "oneOf": [ + { + "type": "object", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Valid values are 5 minutes, 30 minutes, 1 hour, 1 day, n weeks, 1 month, or 1 year.", + "$ref": "#/$defs/slice/string" + }, + "label_col": { + "description": "Column for the label.", + "$ref": "#/$defs/string" + }, + "model_id_col": { + "description": "Column for the model identifier.", + "$ref": "#/$defs/string" + }, + "prediction_col": { + "description": "Column for the prediction.", + "$ref": "#/$defs/string" + }, + "prediction_proba_col": { + "description": "Column for prediction probabilities", + "$ref": "#/$defs/string" + }, + "problem_type": { + "description": "Problem type the model aims to solve.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorInferenceLogProblemType" + }, + "timestamp_col": { + "description": "Column for the timestamp.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "model_id_col", + "prediction_col", + "problem_type", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorInferenceLogProblemType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "PROBLEM_TYPE_CLASSIFICATION", + "PROBLEM_TYPE_REGRESSION" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorMetric": { + "oneOf": [ + { + "type": "object", + "description": "Custom metric definition.", + "properties": { + "definition": { + "description": "Jinja template for a SQL expression that specifies how to compute the metric. See [create metric definition](https://docs.databricks.com/en/lakehouse-monitoring/custom-metrics.html#create-definition).", + "$ref": "#/$defs/string" + }, + "input_columns": { + "description": "A list of column names in the input table the metric should be computed for.\nCan use ``\":table\"`` to indicate that the metric needs information from multiple columns.", + "$ref": "#/$defs/slice/string" + }, + "name": { + "description": "Name of the metric in the output tables.", + "$ref": "#/$defs/string" + }, + "output_data_type": { + "description": "The output type of the custom metric.", + "$ref": "#/$defs/string" + }, + "type": { + "description": "Can only be one of ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"``, ``\"CUSTOM_METRIC_TYPE_DERIVED\"``, or ``\"CUSTOM_METRIC_TYPE_DRIFT\"``.\nThe ``\"CUSTOM_METRIC_TYPE_AGGREGATE\"`` and ``\"CUSTOM_METRIC_TYPE_DERIVED\"`` metrics\nare computed on a single table, whereas the ``\"CUSTOM_METRIC_TYPE_DRIFT\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetricType" + } + }, + "additionalProperties": false, + "required": [ + "definition", + "input_columns", + "name", + "output_data_type", + "type" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorMetricType": { + "oneOf": [ + { + "type": "string", + "description": "Can only be one of ``\\\"CUSTOM_METRIC_TYPE_AGGREGATE\\\"``, ``\\\"CUSTOM_METRIC_TYPE_DERIVED\\\"``, or ``\\\"CUSTOM_METRIC_TYPE_DRIFT\\\"``.\nThe ``\\\"CUSTOM_METRIC_TYPE_AGGREGATE\\\"`` and ``\\\"CUSTOM_METRIC_TYPE_DERIVED\\\"`` metrics\nare computed on a single table, whereas the ``\\\"CUSTOM_METRIC_TYPE_DRIFT\\\"`` compare metrics across\nbaseline and input table, or across the two consecutive time windows.\n- CUSTOM_METRIC_TYPE_AGGREGATE: only depend on the existing columns in your table\n- CUSTOM_METRIC_TYPE_DERIVED: depend on previously computed aggregate metrics\n- CUSTOM_METRIC_TYPE_DRIFT: depend on previously computed aggregate or derived metrics", + "enum": [ + "CUSTOM_METRIC_TYPE_AGGREGATE", + "CUSTOM_METRIC_TYPE_DERIVED", + "CUSTOM_METRIC_TYPE_DRIFT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorNotifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "on_failure": { + "description": "Destinations to send notifications on failure/timeout.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination" + }, + "on_new_classification_tag_detected": { + "description": "Destinations to send notifications on new classification tag detected.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorDestination", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorSnapshot": { + "oneOf": [ + { + "type": "object", + "description": "Snapshot analysis configuration", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorTimeSeries": { + "oneOf": [ + { + "type": "object", + "description": "Time series analysis configuration.", + "properties": { + "granularities": { + "description": "Granularities for aggregating data into time windows based on their timestamp. Valid values are 5 minutes, 30 minutes, 1 hour, 1 day, n weeks, 1 month, or 1 year.", + "$ref": "#/$defs/slice/string" + }, + "timestamp_col": { + "description": "Column for the timestamp.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "granularities", + "timestamp_col" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.VolumeType": { + "oneOf": [ + { + "type": "string", + "description": "The type of the volume. An external volume is located in the specified external location. A managed volume is located in the default location which is specified by the parent schema, or the parent catalog, or the Metastore. [Learn more](https://docs.databricks.com/aws/en/volumes/managed-vs-external)", + "enum": [ + "EXTERNAL", + "MANAGED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Adlsgen2Info": { + "oneOf": [ + { + "type": "object", + "description": "A storage location in Adls Gen2", + "properties": { + "destination": { + "description": "abfss destination, e.g. `abfss://\u003ccontainer-name\u003e@\u003cstorage-account-name\u003e.dfs.core.windows.net/\u003cdirectory-name\u003e`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AutoScale": { + "oneOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded.\nNote that `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers to which the cluster can scale down when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AwsAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during cluster creation which are related to Amazon Web Services.", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAvailability" + }, + "ebs_volume_count": { + "description": "The number of volumes launched for each instance. Users can choose up to 10 volumes.\nThis feature is only enabled for supported node types. Legacy node types cannot specify\ncustom EBS volumes.\nFor node types with no instance store, at least one EBS volume needs to be specified;\notherwise, cluster creation will fail.\n\nThese EBS volumes will be mounted at `/ebs0`, `/ebs1`, and etc.\nInstance store volumes will be mounted at `/local_disk0`, `/local_disk1`, and etc.\n\nIf EBS volumes are attached, Databricks will configure Spark to use only the EBS volumes for\nscratch storage because heterogenously sized scratch devices can lead to inefficient disk\nutilization. If no EBS volumes are attached, Databricks will configure Spark to use instance\nstore volumes.\n\nPlease note that if EBS volumes are specified, then the Spark configuration `spark.local.dir`\nwill be overridden.", + "$ref": "#/$defs/int" + }, + "ebs_volume_iops": { + "description": "If using gp3 volumes, what IOPS to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_size": { + "description": "The size of each EBS volume (in GiB) launched for each instance. For general purpose\nSSD, this value must be within the range 100 - 4096. For throughput optimized HDD,\nthis value must be within the range 500 - 4096.", + "$ref": "#/$defs/int" + }, + "ebs_volume_throughput": { + "description": "If using gp3 volumes, what throughput to use for the disk. If this is not set, the maximum performance of a gp2 volume with the same volume size will be used.", + "$ref": "#/$defs/int" + }, + "ebs_volume_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.EbsVolumeType" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nIf this value is greater than 0, the cluster driver node in particular will be placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "instance_profile_arn": { + "description": "Nodes for this cluster will only be placed on AWS instances with this instance profile. If\nommitted, nodes will be placed on instances without an IAM instance profile. The instance\nprofile must have previously been added to the Databricks environment by an account\nadministrator.\n\nThis feature may only be available to certain customer plans.", + "$ref": "#/$defs/string" + }, + "spot_bid_price_percent": { + "description": "The bid price for AWS spot instances, as a percentage of the corresponding instance type's\non-demand price.\nFor example, if this field is set to 50, and the cluster needs a new `r3.xlarge` spot\ninstance, then the bid price is half of the price of\non-demand `r3.xlarge` instances. Similarly, if this field is set to 200, the bid price is twice\nthe price of on-demand `r3.xlarge` instances. If not specified, the default value is 100.\nWhen spot instances are requested for this cluster, only spot instances whose bid price\npercentage matches this field will be considered.\nNote that, for safety, we enforce this field to be no more than 10000.", + "$ref": "#/$defs/int" + }, + "zone_id": { + "description": "Identifier for the availability zone/datacenter in which the cluster resides.\nThis string will be of a form like \"us-west-2a\". The provided availability\nzone must be in the same region as the Databricks deployment. For example, \"us-west-2a\"\nis not a valid zone id if the Databricks deployment resides in the \"us-east-1\" region.\nThis is an optional field at cluster creation, and if not specified, a default zone will be used.\nIf the zone specified is \"auto\", will try to place cluster in a zone with high availability,\nand will retry placement in a different AZ if there is not enough capacity.\n\nThe list of available zones as well as the default value can be found by using the\n`List Zones` method.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AwsAvailability": { + "oneOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\n\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.", + "enum": [ + "SPOT", + "ON_DEMAND", + "SPOT_WITH_FALLBACK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AzureAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during cluster creation which are related to Microsoft Azure.", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAvailability" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "log_analytics_info": { + "description": "Defines values necessary to configure and run Azure Log Analytics agent", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo" + }, + "spot_bid_max_price": { + "description": "The max bid price to be used for Azure spot instances.\nThe Max price for the bid cannot be higher than the on-demand price of the instance.\nIf not specified, the default value is -1, which specifies that the instance cannot be evicted\non the basis of price, and only on the basis of availability. Further, the value should \u003e 0 or -1.", + "$ref": "#/$defs/float64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.AzureAvailability": { + "oneOf": [ + { + "type": "string", + "description": "Availability type used for all subsequent nodes past the `first_on_demand` ones.\nNote: If `first_on_demand` is zero, this availability type will be used for the entire cluster.", + "enum": [ + "SPOT_AZURE", + "ON_DEMAND_AZURE", + "SPOT_WITH_FALLBACK_AZURE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.ClientsTypes": { + "oneOf": [ + { + "type": "object", + "properties": { + "jobs": { + "description": "With jobs set, the cluster can be used for jobs", + "$ref": "#/$defs/bool" + }, + "notebooks": { + "description": "With notebooks set, this cluster can be used for notebooks", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.ClusterLogConf": { + "oneOf": [ + { + "type": "object", + "description": "Cluster log delivery config", + "properties": { + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\" : { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \"s3\": { \"destination\" : \"s3://cluster_log_bucket/prefix\", \"region\" : \"us-west-2\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + }, + "volumes": { + "description": "destination needs to be provided, e.g.\n`{ \"volumes\": { \"destination\": \"/Volumes/catalog/schema/volume/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.ClusterSpec": { + "oneOf": [ + { + "type": "object", + "description": "Contains a snapshot of the latest user specified settings that were used to create/edit the cluster.", + "properties": { + "apply_policy_default_values": { + "description": "When set to true, fixed and default values from the policy will be used for fields that are omitted. When set to false, only fixed values from the policy will be applied.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AutoScale" + }, + "autotermination_minutes": { + "description": "Automatically terminates the cluster after it is inactive for this time in minutes. If not set,\nthis cluster will not be automatically terminated. If specified, the threshold must be between\n10 and 10000 minutes.\nUsers can also set this value to 0 to explicitly disable automatic termination.", + "$ref": "#/$defs/int" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nThree kinds of destinations (DBFS, S3 and Unity Catalog volumes) are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "cluster_name": { + "description": "Cluster name requested by the user. This doesn't have to be unique.\nIf not specified at creation, the cluster name will be an empty string.\nFor job clusters, the cluster name is automatically set based on the job and job run IDs.", + "$ref": "#/$defs/string" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "data_security_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DataSecurityMode" + }, + "docker_image": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerImage" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above.\n\nThis field, along with node_type_id, should not be set if virtual_cluster_size is set.\nIf both driver_node_type_id, node_type_id, and virtual_cluster_size are specified, driver_node_type_id and node_type_id take precedence.", + "$ref": "#/$defs/string" + }, + "enable_elastic_disk": { + "description": "Autoscaling Local Storage: when enabled, this cluster will dynamically acquire additional disk\nspace when its Spark workers are running low on disk space. This feature requires specific AWS\npermissions to function correctly - refer to the User Guide for more details.", + "$ref": "#/$defs/bool" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable LUKS on cluster VMs' local disks", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified.\nThe scripts are executed sequentially in the order provided.\nIf `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "is_single_node": { + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\nWhen set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`", + "$ref": "#/$defs/bool" + }, + "kind": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Kind" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "remote_disk_throughput": { + "description": "If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported for GCP HYPERDISK_BALANCED disks.", + "$ref": "#/$defs/int" + }, + "runtime_engine": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RuntimeEngine" + }, + "single_user_name": { + "description": "Single user name if data_security_mode is `SINGLE_USER`", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nUsers can also pass in a string of extra JVM options to the driver and the executors via\n`spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively.", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "spark_version": { + "description": "The Spark version of the cluster, e.g. `3.3.x-scala2.11`.\nA list of available Spark versions can be retrieved by using\nthe :method:clusters/sparkVersions API call.", + "$ref": "#/$defs/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + }, + "total_initial_remote_disk_size": { + "description": "If set, what the total initial volume size (in GB) of the remote disks should be. Currently only supported for GCP HYPERDISK_BALANCED disks.", + "$ref": "#/$defs/int" + }, + "use_ml_runtime": { + "description": "This field can only be used when `kind = CLASSIC_PREVIEW`.\n\n`effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.", + "$ref": "#/$defs/bool" + }, + "workload_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DataSecurityMode": { + "oneOf": [ + { + "type": "string", + "description": "Data security mode decides what data governance model to use when accessing data\nfrom a cluster.\n\nThe following modes can only be used when `kind = CLASSIC_PREVIEW`.\n* `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration.\n* `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`.\n* `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`.\n\nThe following modes can be used regardless of `kind`.\n* `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode.\n* `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode.\n* `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited.\n\nThe following modes are deprecated starting with Databricks Runtime 15.0 and\nwill be removed for future Databricks Runtime versions:\n\n* `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters.\n* `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters.\n* `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters.\n* `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.", + "enum": [ + "NONE", + "SINGLE_USER", + "USER_ISOLATION", + "LEGACY_TABLE_ACL", + "LEGACY_PASSTHROUGH", + "LEGACY_SINGLE_USER", + "LEGACY_SINGLE_USER_STANDARD", + "DATA_SECURITY_MODE_STANDARD", + "DATA_SECURITY_MODE_DEDICATED", + "DATA_SECURITY_MODE_AUTO" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DbfsStorageInfo": { + "oneOf": [ + { + "type": "object", + "description": "A storage location in DBFS", + "properties": { + "destination": { + "description": "dbfs destination, e.g. `dbfs:/my/path`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DockerBasicAuth": { + "oneOf": [ + { + "type": "object", + "properties": { + "password": { + "description": "Password of the user", + "$ref": "#/$defs/string" + }, + "username": { + "description": "Name of the user", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.DockerImage": { + "oneOf": [ + { + "type": "object", + "properties": { + "basic_auth": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DockerBasicAuth" + }, + "url": { + "description": "URL of the docker image.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.EbsVolumeType": { + "oneOf": [ + { + "type": "string", + "description": "All EBS volume types that Databricks supports.\nSee https://aws.amazon.com/ebs/details/ for details.", + "enum": [ + "GENERAL_PURPOSE_SSD", + "THROUGHPUT_OPTIMIZED_HDD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Environment": { + "oneOf": [ + { + "type": "object", + "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip dependencies are supported.", + "properties": { + "client": { + "description": "Use `environment_version` instead.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "dependencies": { + "description": "List of pip dependencies, as supported by the version of pip in this environment.", + "$ref": "#/$defs/slice/string" + }, + "environment_version": { + "description": "Required. Environment version used by the environment.\nEach version comes with a specific Python version and a set of Python packages.\nThe version is a string, consisting of an integer.", + "$ref": "#/$defs/string" + }, + "jar_dependencies": { + "description": "List of jar dependencies, should be string representing volume paths. For example: `/Volumes/path/to/test.jar`.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.GcpAttributes": { + "oneOf": [ + { + "type": "object", + "description": "Attributes set during cluster creation which are related to GCP.", + "properties": { + "availability": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAvailability" + }, + "boot_disk_size": { + "description": "Boot disk size in GB", + "$ref": "#/$defs/int" + }, + "first_on_demand": { + "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", + "$ref": "#/$defs/int" + }, + "google_service_account": { + "description": "If provided, the cluster will impersonate the google service account when accessing\ngcloud services (like GCS). The google service account\nmust have previously been added to the Databricks environment by an account\nadministrator.", + "$ref": "#/$defs/string" + }, + "local_ssd_count": { + "description": "If provided, each node (workers and driver) in the cluster will have this number of local SSDs attached.\nEach local SSD is 375GB in size.\nRefer to [GCP documentation](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds)\nfor the supported number of local SSDs for each instance type.", + "$ref": "#/$defs/int" + }, + "use_preemptible_executors": { + "description": "This field determines whether the spark executors will be scheduled to run on preemptible\nVMs (when set to true) versus standard compute engine VMs (when set to false; default).\nNote: Soon to be deprecated, use the 'availability' field instead.", + "$ref": "#/$defs/bool", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "zone_id": { + "description": "Identifier for the availability zone in which the cluster resides.\nThis can be one of the following:\n- \"HA\" =\u003e High availability, spread nodes across availability zones for a Databricks deployment region [default].\n- \"AUTO\" =\u003e Databricks picks an availability zone to schedule the cluster on.\n- A GCP availability zone =\u003e Pick One of the available zones for (machine type + region) from\nhttps://cloud.google.com/compute/docs/regions-zones.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.GcpAvailability": { + "oneOf": [ + { + "type": "string", + "description": "This field determines whether the instance pool will contain preemptible\nVMs, on-demand VMs, or preemptible VMs with a fallback to on-demand VMs if the former is unavailable.", + "enum": [ + "PREEMPTIBLE_GCP", + "ON_DEMAND_GCP", + "PREEMPTIBLE_WITH_FALLBACK_GCP" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.GcsStorageInfo": { + "oneOf": [ + { + "type": "object", + "description": "A storage location in Google Cloud Platform's GCS", + "properties": { + "destination": { + "description": "GCS destination/URI, e.g. `gs://my-bucket/some-prefix`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "oneOf": [ + { + "type": "object", + "description": "Config for an individual init script\nNext ID: 11", + "properties": { + "abfss": { + "description": "Contains the Azure Data Lake Storage destination path", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Adlsgen2Info" + }, + "dbfs": { + "description": "destination needs to be provided. e.g.\n`{ \"dbfs\": { \"destination\" : \"dbfs:/home/cluster_log\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.DbfsStorageInfo", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "file": { + "description": "destination needs to be provided, e.g.\n`{ \"file\": { \"destination\": \"file:/my/local/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.LocalFileInfo" + }, + "gcs": { + "description": "destination needs to be provided, e.g.\n`{ \"gcs\": { \"destination\": \"gs://my-bucket/file.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcsStorageInfo" + }, + "s3": { + "description": "destination and either the region or endpoint need to be provided. e.g.\n`{ \\\"s3\\\": { \\\"destination\\\": \\\"s3://cluster_log_bucket/prefix\\\", \\\"region\\\": \\\"us-west-2\\\" } }`\nCluster iam role is used to access s3, please make sure the cluster iam role in\n`instance_profile_arn` has permission to write data to the s3 destination.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.S3StorageInfo" + }, + "volumes": { + "description": "destination needs to be provided. e.g.\n`{ \\\"volumes\\\" : { \\\"destination\\\" : \\\"/Volumes/my-init.sh\\\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.VolumesStorageInfo" + }, + "workspace": { + "description": "destination needs to be provided, e.g.\n`{ \"workspace\": { \"destination\": \"/cluster-init-scripts/setup-datadog.sh\" } }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.WorkspaceStorageInfo" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Kind": { + "type": "string" + }, + "compute.Library": { + "oneOf": [ + { + "type": "object", + "properties": { + "cran": { + "description": "Specification of a CRAN library to be installed as part of the library", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.RCranLibrary" + }, + "egg": { + "description": "Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "jar": { + "description": "URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"jar\": \"/Workspace/path/to/library.jar\" }`, `{ \"jar\" : \"/Volumes/path/to/library.jar\" }` or\n`{ \"jar\": \"s3://my-bucket/library.jar\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + }, + "maven": { + "description": "Specification of a maven library to be installed. For example:\n`{ \"coordinates\": \"org.jsoup:jsoup:1.7.2\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary" + }, + "pypi": { + "description": "Specification of a PyPi library to be installed. For example:\n`{ \"package\": \"simplejson\" }`", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.PythonPyPiLibrary" + }, + "requirements": { + "description": "URI of the requirements.txt file to install. Only Workspace paths and Unity Catalog Volumes paths are supported.\nFor example: `{ \"requirements\": \"/Workspace/path/to/requirements.txt\" }` or `{ \"requirements\" : \"/Volumes/path/to/requirements.txt\" }`", + "$ref": "#/$defs/string" + }, + "whl": { + "description": "URI of the wheel library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.\nFor example: `{ \"whl\": \"/Workspace/path/to/library.whl\" }`, `{ \"whl\" : \"/Volumes/path/to/library.whl\" }` or\n`{ \"whl\": \"s3://my-bucket/library.whl\" }`.\nIf S3 is used, please make sure the cluster has read access on the library. You may need to\nlaunch the cluster with an IAM role to access the S3 URI.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.LocalFileInfo": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination": { + "description": "local file destination, e.g. `file:/my/local/file.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.LogAnalyticsInfo": { + "oneOf": [ + { + "type": "object", + "properties": { + "log_analytics_primary_key": { + "description": "The primary key for the Azure Log Analytics agent configuration", + "$ref": "#/$defs/string" + }, + "log_analytics_workspace_id": { + "description": "The workspace ID for the Azure Log Analytics agent configuration", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.MavenLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "coordinates": { + "description": "Gradle-style maven coordinates. For example: \"org.jsoup:jsoup:1.7.2\".", + "$ref": "#/$defs/string" + }, + "exclusions": { + "description": "List of dependences to exclude. For example: `[\"slf4j:slf4j\", \"*:hadoop-client\"]`.\n\nMaven dependency exclusions:\nhttps://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html.", + "$ref": "#/$defs/slice/string" + }, + "repo": { + "description": "Maven repo to install the Maven package from. If omitted, both Maven Central Repository\nand Spark Packages are searched.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "coordinates" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.PythonPyPiLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the pypi package to install. An optional exact version specification is also\nsupported. Examples: \"simplejson\" and \"simplejson==3.8.0\".", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default pip index is\nused.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.RCranLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "package": { + "description": "The name of the CRAN package to install.", + "$ref": "#/$defs/string" + }, + "repo": { + "description": "The repository where the package can be found. If not specified, the default CRAN repo is used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "package" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.RuntimeEngine": { + "oneOf": [ + { + "type": "string", + "enum": [ + "NULL", + "STANDARD", + "PHOTON" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.S3StorageInfo": { + "oneOf": [ + { + "type": "object", + "description": "A storage location in Amazon S3", + "properties": { + "canned_acl": { + "description": "(Optional) Set canned access control list for the logs, e.g. `bucket-owner-full-control`.\nIf `canned_cal` is set, please make sure the cluster iam role has `s3:PutObjectAcl` permission on\nthe destination bucket and prefix. The full list of possible canned acl can be found at\nhttp://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl.\nPlease also note that by default only the object owner gets full controls. If you are using cross account\nrole for writing data, you may want to set `bucket-owner-full-control` to make bucket owner able to\nread the logs.", + "$ref": "#/$defs/string" + }, + "destination": { + "description": "S3 destination, e.g. `s3://my-bucket/some-prefix` Note that logs will be delivered using\ncluster iam role, please make sure you set cluster iam role and the role has write access to the\ndestination. Please also note that you cannot use AWS keys to deliver logs.", + "$ref": "#/$defs/string" + }, + "enable_encryption": { + "description": "(Optional) Flag to enable server side encryption, `false` by default.", + "$ref": "#/$defs/bool" + }, + "encryption_type": { + "description": "(Optional) The encryption type, it could be `sse-s3` or `sse-kms`. It will be used only when\nencryption is enabled and the default type is `sse-s3`.", + "$ref": "#/$defs/string" + }, + "endpoint": { + "description": "S3 endpoint, e.g. `https://s3-us-west-2.amazonaws.com`. Either region or endpoint needs to be set.\nIf both are set, endpoint will be used.", + "$ref": "#/$defs/string" + }, + "kms_key": { + "description": "(Optional) Kms key which will be used if encryption is enabled and encryption type is set to `sse-kms`.", + "$ref": "#/$defs/string" + }, + "region": { + "description": "S3 region, e.g. `us-west-2`. Either region or endpoint needs to be set. If both are set,\nendpoint will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.VolumesStorageInfo": { + "oneOf": [ + { + "type": "object", + "description": "A storage location back by UC Volumes.", + "properties": { + "destination": { + "description": "UC Volumes destination, e.g. `/Volumes/catalog/schema/vol1/init-scripts/setup-datadog.sh`\nor `dbfs:/Volumes/catalog/schema/vol1/init-scripts/setup-datadog.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.WorkloadType": { + "oneOf": [ + { + "type": "object", + "description": "Cluster Attributes showing for clusters workload types.", + "properties": { + "clients": { + "description": "defined what type of clients can use the cluster. E.g. Notebooks, Jobs", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClientsTypes" + } + }, + "additionalProperties": false, + "required": [ + "clients" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.WorkspaceStorageInfo": { + "oneOf": [ + { + "type": "object", + "description": "A storage location in Workspace Filesystem (WSFS)", + "properties": { + "destination": { + "description": "wsfs destination, e.g. `workspace:/cluster-init-scripts/setup-datadog.sh`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "destination" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "dashboards.LifecycleState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "ACTIVE", + "TRASHED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.DatabaseInstanceRef": { + "oneOf": [ + { + "type": "object", + "description": "DatabaseInstanceRef is a reference to a database instance. It is used in the\nDatabaseInstance object to refer to the parent instance of an instance and\nto refer the child instances of an instance.\nTo specify as a parent instance during creation of an instance,\nthe lsn and branch_time fields are optional. If not specified, the child\ninstance will be created from the latest lsn of the parent.\nIf both lsn and branch_time are specified, the lsn will be used to create\nthe child instance.", + "properties": { + "branch_time": { + "description": "Branch time of the ref database instance.\nFor a parent ref instance, this is the point in time on the parent instance from which the\ninstance was created.\nFor a child ref instance, this is the point in time on the instance from which the child\ninstance was created.\nInput: For specifying the point in time to create a child instance. Optional.\nOutput: Only populated if provided as input to create a child instance.", + "$ref": "#/$defs/string" + }, + "effective_lsn": { + "description": "xref AIP-129. `lsn` is owned by the client, while `effective_lsn` is owned by the server.\n`lsn` will only be set in Create/Update response messages if and only if the user provides the field via the request.\n`effective_lsn` on the other hand will always bet set in all response messages (Create/Update/Get/List).\nFor a parent ref instance, this is the LSN on the parent instance from which the\ninstance was created.\nFor a child ref instance, this is the LSN on the instance from which the child instance\nwas created.", + "$ref": "#/$defs/string" + }, + "lsn": { + "description": "User-specified WAL LSN of the ref database instance.\n\nInput: For specifying the WAL LSN to create a child instance. Optional.\nOutput: Only populated if provided as input to create a child instance.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "Name of the ref database instance.", + "$ref": "#/$defs/string" + }, + "uid": { + "description": "Id of the ref database instance.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.DatabaseInstanceState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "STARTING", + "AVAILABLE", + "DELETING", + "STOPPED", + "UPDATING", + "FAILING_OVER" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.DeltaTableSyncInfo": { + "oneOf": [ + { + "type": "object", + "properties": { + "delta_commit_timestamp": { + "$ref": "#/$defs/string" + }, + "delta_commit_version": { + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.NewPipelineSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "storage_catalog": { + "$ref": "#/$defs/string" + }, + "storage_schema": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.ProvisioningInfoState": { + "type": "string" + }, + "database.ProvisioningPhase": { + "type": "string" + }, + "database.SyncedTableContinuousUpdateStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "initial_pipeline_sync_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + }, + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableFailedStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTablePipelineProgress": { + "oneOf": [ + { + "type": "object", + "properties": { + "estimated_completion_time_seconds": { + "$ref": "#/$defs/float64" + }, + "latest_version_currently_processing": { + "$ref": "#/$defs/int64" + }, + "provisioning_phase": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.ProvisioningPhase" + }, + "sync_progress_completion": { + "$ref": "#/$defs/float64" + }, + "synced_row_count": { + "$ref": "#/$defs/int64" + }, + "total_row_count": { + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTablePosition": { + "oneOf": [ + { + "type": "object", + "properties": { + "delta_table_sync_info": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.DeltaTableSyncInfo" + }, + "sync_end_timestamp": { + "$ref": "#/$defs/string" + }, + "sync_start_timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableProvisioningStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "initial_pipeline_sync_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableSchedulingPolicy": { + "type": "string" + }, + "database.SyncedTableSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "create_database_objects_if_missing": { + "$ref": "#/$defs/bool" + }, + "existing_pipeline_id": { + "$ref": "#/$defs/string" + }, + "new_pipeline_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.NewPipelineSpec" + }, + "primary_key_columns": { + "$ref": "#/$defs/slice/string" + }, + "scheduling_policy": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableSchedulingPolicy" + }, + "source_table_full_name": { + "$ref": "#/$defs/string" + }, + "timeseries_key": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableState": { + "type": "string" + }, + "database.SyncedTableStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "continuous_update_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableContinuousUpdateStatus" + }, + "detailed_state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableState" + }, + "failed_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableFailedStatus" + }, + "last_sync": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePosition" + }, + "message": { + "$ref": "#/$defs/string" + }, + "pipeline_id": { + "$ref": "#/$defs/string" + }, + "provisioning_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableProvisioningStatus" + }, + "triggered_update_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableTriggeredUpdateStatus" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableTriggeredUpdateStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + }, + "triggered_update_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.AuthenticationMethod": { + "oneOf": [ + { + "type": "string", + "enum": [ + "OAUTH", + "PAT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.CleanRoomsNotebookTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "clean_room_name": { + "description": "The clean room that the notebook belongs to.", + "$ref": "#/$defs/string" + }, + "etag": { + "description": "Checksum to validate the freshness of the notebook resource (i.e. the notebook being run is the latest version).\nIt can be fetched by calling the :method:cleanroomassets/get API.", + "$ref": "#/$defs/string" + }, + "notebook_base_parameters": { + "description": "Base parameters to be used for the clean room notebook job.", + "$ref": "#/$defs/map/string" + }, + "notebook_name": { + "description": "Name of the notebook being run.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "clean_room_name", + "notebook_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ComputeConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "gpu_node_pool_id": { + "description": "IDof the GPU pool to use.", + "$ref": "#/$defs/string" + }, + "gpu_type": { + "description": "GPU type.", + "$ref": "#/$defs/string" + }, + "num_gpus": { + "description": "Number of GPUs.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "num_gpus" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Condition": { + "oneOf": [ + { + "type": "string", + "enum": [ + "ANY_UPDATED", + "ALL_UPDATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ConditionTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "left": { + "description": "The left operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + }, + "op": { + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTaskOp" + }, + "right": { + "description": "The right operand of the condition task. Can be either a string value or a job state or parameter reference.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "left", + "op", + "right" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ConditionTaskOp": { + "oneOf": [ + { + "type": "string", + "description": "* `EQUAL_TO`, `NOT_EQUAL` operators perform string comparison of their operands. This means that `“12.0” == “12”` will evaluate to `false`.\n* `GREATER_THAN`, `GREATER_THAN_OR_EQUAL`, `LESS_THAN`, `LESS_THAN_OR_EQUAL` operators perform numeric comparison of their operands. `“12.0” \u003e= “12”` will evaluate to `true`, `“10.0” \u003e= “12”` will evaluate to `false`.\n\nThe boolean comparison to task values can be implemented with operators `EQUAL_TO`, `NOT_EQUAL`. If a task value was set to a boolean value, it will be serialized to `“true”` or `“false”` for the comparison.", + "enum": [ + "EQUAL_TO", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "NOT_EQUAL" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Continuous": { + "oneOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.CronSchedule": { + "oneOf": [ + { + "type": "object", + "properties": { + "pause_status": { + "description": "Indicate whether this schedule is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "quartz_cron_expression": { + "description": "A Cron expression using Quartz syntax that describes the schedule for a job. See [Cron Trigger](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) for details. This field is required.", + "$ref": "#/$defs/string" + }, + "timezone_id": { + "description": "A Java timezone ID. The schedule for a job is resolved with respect to this timezone. See [Java TimeZone](https://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html) for details. This field is required.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "quartz_cron_expression", + "timezone_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.DashboardTask": { + "oneOf": [ + { + "type": "object", + "description": "Configures the Lakeview Dashboard job task type.", + "properties": { + "dashboard_id": { + "$ref": "#/$defs/string" + }, + "subscription": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Subscription" + }, + "warehouse_id": { + "description": "Optional: The warehouse id to execute the dashboard with for the schedule.\nIf not specified, the default warehouse of the dashboard will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.DbtCloudTask": { + "oneOf": [ + { + "type": "object", + "description": "Deprecated in favor of DbtPlatformTask", + "properties": { + "connection_resource_name": { + "description": "The resource name of the UC connection that authenticates the dbt Cloud for this task", + "$ref": "#/$defs/string" + }, + "dbt_cloud_job_id": { + "description": "Id of the dbt Cloud job to be triggered", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.DbtPlatformTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection_resource_name": { + "description": "The resource name of the UC connection that authenticates the dbt platform for this task", + "$ref": "#/$defs/string" + }, + "dbt_platform_job_id": { + "description": "Id of the dbt platform job to be triggered. Specified as a string for maximum compatibility with clients.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.DbtTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "Optional name of the catalog to use. The value is the top level in the 3-level namespace of Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a warehouse_id is specified. Requires dbt-databricks \u003e= 1.1.1.", + "$ref": "#/$defs/string" + }, + "commands": { + "description": "A list of dbt commands to execute. All commands must start with `dbt`. This parameter must not be empty. A maximum of up to 10 commands can be provided.", + "$ref": "#/$defs/slice/string" + }, + "profiles_directory": { + "description": "Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is specified. If no warehouse_id is specified and this folder is unset, the root directory is used.", + "$ref": "#/$defs/string" + }, + "project_directory": { + "description": "Path to the project directory. Optional for Git sourced tasks, in which\ncase if no value is provided, the root of the Git repository is used.", + "$ref": "#/$defs/string" + }, + "schema": { + "description": "Optional schema to write to. This parameter is only used when a warehouse_id is also provided. If not provided, the `default` schema is used.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the project directory. When set to `WORKSPACE`, the project will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the project will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: Project is located in Databricks workspace.\n* `GIT`: Project is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the profile and connection details to dbt. It can be overridden on a per-command basis by using the `--profiles-dir` command line argument.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "commands" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.FileArrivalTriggerConfiguration": { + "oneOf": [ + { + "type": "object", + "properties": { + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds", + "$ref": "#/$defs/int" + }, + "url": { + "description": "URL to be monitored for file arrivals. The path must point to the root or a subpath of the external location.", + "$ref": "#/$defs/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no file activity has occurred for the specified amount of time.\nThis makes it possible to wait for a batch of incoming files to arrive before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.ForEachTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "concurrency": { + "description": "An optional maximum allowed number of concurrent runs of the task.\nSet this value if you want to be able to execute multiple runs of the task concurrently.", + "$ref": "#/$defs/int" + }, + "inputs": { + "description": "Array for task to iterate on. This can be a JSON string or a reference to\nan array parameter.", + "$ref": "#/$defs/string" + }, + "task": { + "description": "Configuration for the task that will be run for each element in the array", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + "additionalProperties": false, + "required": [ + "inputs", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Format": { + "oneOf": [ + { + "type": "string", + "enum": [ + "SINGLE_TASK", + "MULTI_TASK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.GenAiComputeTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "command": { + "description": "Command launcher to run the actual script, e.g. bash, python etc.", + "$ref": "#/$defs/string" + }, + "compute": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ComputeConfig" + }, + "dl_runtime_image": { + "description": "Runtime image", + "$ref": "#/$defs/string" + }, + "mlflow_experiment_name": { + "description": "Optional string containing the name of the MLflow experiment to log the run to. If name is not\nfound, backend will create the mlflow experiment using the name.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the training script. When set to `WORKSPACE`, the script will be retrieved from the local Databricks workspace. When set to `GIT`, the script will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Script is located in Databricks workspace.\n* `GIT`: Script is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "training_script_path": { + "description": "The training script file path to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "yaml_parameters": { + "description": "Optional string containing model parameters passed to the training script in yaml format.\nIf present, then the content in yaml_parameters_file_path will be ignored.", + "$ref": "#/$defs/string" + }, + "yaml_parameters_file_path": { + "description": "Optional path to a YAML file containing model parameters passed to the training script.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "dl_runtime_image" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.GitProvider": { + "oneOf": [ + { + "type": "string", + "enum": [ + "gitHub", + "bitbucketCloud", + "azureDevOpsServices", + "gitHubEnterprise", + "bitbucketServer", + "gitLab", + "gitLabEnterpriseEdition", + "awsCodeCommit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.GitSnapshot": { + "oneOf": [ + { + "type": "object", + "description": "Read-only state of the remote repository at the time the job was run. This field is only included on job runs.", + "properties": { + "used_commit": { + "description": "Commit that was used to execute the run. If git_branch was specified, this points to the HEAD of the branch at the time of the run; if git_tag was specified, this points to the commit the tag points to.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.GitSource": { + "oneOf": [ + { + "type": "object", + "description": "An optional specification for a remote Git repository containing the source code used by tasks. Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.\n\nIf `git_source` is set, these tasks retrieve the file from the remote repository by default. However, this behavior can be overridden by setting `source` to `WORKSPACE` on the task.\n\nNote: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks are used, `git_source` must be defined on the job.", + "properties": { + "git_branch": { + "description": "Name of the branch to be checked out and used by this job. This field cannot be specified in conjunction with git_tag or git_commit.", + "$ref": "#/$defs/string" + }, + "git_commit": { + "description": "Commit to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_tag.", + "$ref": "#/$defs/string" + }, + "git_provider": { + "description": "Unique identifier of the service used to host the Git repository. The value is case insensitive.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GitProvider" + }, + "git_tag": { + "description": "Name of the tag to be checked out and used by this job. This field cannot be specified in conjunction with git_branch or git_commit.", + "$ref": "#/$defs/string" + }, + "git_url": { + "description": "URL of the repository to be cloned by this job.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "git_provider", + "git_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobCluster": { + "oneOf": [ + { + "type": "object", + "properties": { + "job_cluster_key": { + "description": "A unique name for the job cluster. This field is required and must be unique within the job.\n`JobTaskSettings` may refer to this field to determine which cluster to launch for the task execution.", + "$ref": "#/$defs/string" + }, + "new_cluster": { + "description": "If new_cluster, a description of a cluster that is created for each task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + } + }, + "additionalProperties": false, + "required": [ + "job_cluster_key", + "new_cluster" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobDeployment": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The kind of deployment that manages the job.\n\n* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobDeploymentKind" + }, + "metadata_file_path": { + "description": "Path of the file that contains deployment metadata.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "kind" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobDeploymentKind": { + "oneOf": [ + { + "type": "string", + "description": "* `BUNDLE`: The job is managed by Databricks Asset Bundle.", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEditMode": { + "oneOf": [ + { + "type": "string", + "description": "Edit mode of the job.\n\n* `UI_LOCKED`: The job is in a locked UI state and cannot be modified.\n* `EDITABLE`: The job is in an editable state and can be modified.", + "enum": [ + "UI_LOCKED", + "EDITABLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEmailNotifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", + "$ref": "#/$defs/bool", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment_key": { + "description": "The key of an environment. It has to be unique within a job.", + "$ref": "#/$defs/string" + }, + "spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Environment" + } + }, + "additionalProperties": false, + "required": [ + "environment_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobNotificationSettings": { + "oneOf": [ + { + "type": "object", + "properties": { + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "oneOf": [ + { + "type": "object", + "properties": { + "default": { + "description": "Default value of the parameter.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of the defined parameter. May only contain alphanumeric characters, `_`, `-`, and `.`", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "default", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobRunAs": { + "oneOf": [ + { + "type": "object", + "description": "Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.\n\nEither `user_name` or `service_principal_name` should be specified. If not, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "The application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Non-admin users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobSource": { + "oneOf": [ + { + "type": "object", + "description": "The source of the job specification in the remote repository when the job is source controlled.", + "properties": { + "dirty_state": { + "description": "Dirty state indicates the job is not fully synced with the job specification in the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobSourceDirtyState" + }, + "import_from_git_branch": { + "description": "Name of the branch which the job is imported from.", + "$ref": "#/$defs/string" + }, + "job_config_path": { + "description": "Path of the job YAML file that contains the job specification.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "import_from_git_branch", + "job_config_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobSourceDirtyState": { + "oneOf": [ + { + "type": "string", + "description": "Dirty state indicates the job is not fully synced with the job specification\nin the remote repository.\n\nPossible values are:\n* `NOT_SYNCED`: The job is not yet synced with the remote job specification. Import the remote job specification from UI to make the job fully synced.\n* `DISCONNECTED`: The job is temporary disconnected from the remote job specification and is allowed for live edit. Import the remote job specification again from UI to make the job fully synced.", + "enum": [ + "NOT_SYNCED", + "DISCONNECTED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthMetric": { + "oneOf": [ + { + "type": "string", + "description": "Specifies the health metric that is being evaluated for a particular health rule.\n\n* `RUN_DURATION_SECONDS`: Expected total time for a run in seconds.\n* `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview.\n* `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview.", + "enum": [ + "RUN_DURATION_SECONDS", + "STREAMING_BACKLOG_BYTES", + "STREAMING_BACKLOG_RECORDS", + "STREAMING_BACKLOG_SECONDS", + "STREAMING_BACKLOG_FILES" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthOperator": { + "oneOf": [ + { + "type": "string", + "description": "Specifies the operator used to compare the health metric value with the specified threshold.", + "enum": [ + "GREATER_THAN" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthRule": { + "oneOf": [ + { + "type": "object", + "properties": { + "metric": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthMetric" + }, + "op": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthOperator" + }, + "value": { + "description": "Specifies the threshold value that the health metric should obey to satisfy the health rule.", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false, + "required": [ + "metric", + "op", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthRules": { + "oneOf": [ + { + "type": "object", + "description": "An optional set of health rules that can be defined for this job.", + "properties": { + "rules": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.NotebookTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "base_parameters": { + "description": "Base parameters to be used for each run of this job. If the run is initiated by a call to :method:jobs/run\nNow with parameters specified, the two parameters maps are merged. If the same key is specified in\n`base_parameters` and in `run-now`, the value from `run-now` is used.\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nIf the notebook takes a parameter that is not specified in the job’s `base_parameters` or the `run-now` override parameters,\nthe default value from the notebook is used.\n\nRetrieve these parameters in a notebook using [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-widgets).\n\nThe JSON representation of this field cannot exceed 1MB.", + "$ref": "#/$defs/map/string" + }, + "notebook_path": { + "description": "The path of the notebook to be run in the Databricks workspace or remote repository.\nFor notebooks stored in the Databricks workspace, the path must be absolute and begin with a slash.\nFor notebooks stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the notebook. When set to `WORKSPACE`, the notebook will be retrieved from the local Databricks workspace. When set to `GIT`, the notebook will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n* `WORKSPACE`: Notebook is located in Databricks workspace.\n* `GIT`: Notebook is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + }, + "warehouse_id": { + "description": "Optional `warehouse_id` to run the notebook on a SQL warehouse. Classic SQL warehouses are NOT supported, please use serverless or pro SQL warehouses.\n\nNote that SQL warehouses only support SQL cells; if the notebook contains non-SQL cells, the run will fail.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "notebook_path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PauseStatus": { + "oneOf": [ + { + "type": "string", + "enum": [ + "UNPAUSED", + "PAUSED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PerformanceTarget": { + "oneOf": [ + { + "type": "string", + "description": "PerformanceTarget defines how performant (lower latency) or cost efficient the execution of run on serverless compute should be.\nThe performance mode on the job or pipeline should map to a performance setting that is passed to Cluster Manager\n(see cluster-common PerformanceTarget).", + "enum": [ + "PERFORMANCE_OPTIMIZED", + "STANDARD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PeriodicTriggerConfiguration": { + "oneOf": [ + { + "type": "object", + "properties": { + "interval": { + "description": "The interval at which the trigger should run.", + "$ref": "#/$defs/int" + }, + "unit": { + "description": "The unit of time for the interval.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfigurationTimeUnit" + } + }, + "additionalProperties": false, + "required": [ + "interval", + "unit" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PeriodicTriggerConfigurationTimeUnit": { + "oneOf": [ + { + "type": "string", + "enum": [ + "HOURS", + "DAYS", + "WEEKS" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PipelineParams": { + "oneOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PipelineTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "full_refresh": { + "description": "If true, triggers a full refresh on the delta live table.", + "$ref": "#/$defs/bool" + }, + "pipeline_id": { + "description": "The full name of the pipeline task to execute.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "pipeline_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PowerBiModel": { + "oneOf": [ + { + "type": "object", + "properties": { + "authentication_method": { + "description": "How the published Power BI model authenticates to Databricks", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.AuthenticationMethod" + }, + "model_name": { + "description": "The name of the Power BI model", + "$ref": "#/$defs/string" + }, + "overwrite_existing": { + "description": "Whether to overwrite existing Power BI models", + "$ref": "#/$defs/bool" + }, + "storage_mode": { + "description": "The default storage mode of the Power BI model", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.StorageMode" + }, + "workspace_name": { + "description": "The name of the Power BI workspace of the model", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PowerBiTable": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog": { + "description": "The catalog name in Databricks", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The table name in Databricks", + "$ref": "#/$defs/string" + }, + "schema": { + "description": "The schema name in Databricks", + "$ref": "#/$defs/string" + }, + "storage_mode": { + "description": "The Power BI storage mode of the table", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.StorageMode" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PowerBiTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection_resource_name": { + "description": "The resource name of the UC connection to authenticate from Databricks to Power BI", + "$ref": "#/$defs/string" + }, + "power_bi_model": { + "description": "The semantic model to update", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiModel" + }, + "refresh_after_update": { + "description": "Whether the model should be refreshed after the update", + "$ref": "#/$defs/bool" + }, + "tables": { + "description": "The tables to be exported to Power BI", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTable" + }, + "warehouse_id": { + "description": "The SQL warehouse ID to use as the Power BI data source", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PythonWheelTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "entry_point": { + "description": "Named entry point to use, if it does not exist in the metadata of the package it executes the function from the package directly using `$packageName.$entryPoint()`", + "$ref": "#/$defs/string" + }, + "named_parameters": { + "description": "Command-line parameters passed to Python wheel task in the form of `[\"--name=task\", \"--data=dbfs:/path/to/data.json\"]`. Leave it empty if `parameters` is not null.", + "$ref": "#/$defs/map/string" + }, + "package_name": { + "description": "Name of the package to execute", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Command-line parameters passed to Python wheel task. Leave it empty if `named_parameters` is not null.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false, + "required": [ + "entry_point", + "package_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.QueueSettings": { + "oneOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "If true, enable queueing for the job. This is a required field.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false, + "required": [ + "enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.RunIf": { + "oneOf": [ + { + "type": "string", + "description": "An optional value indicating the condition that determines whether the task should be run once its dependencies have been completed. When omitted, defaults to `ALL_SUCCESS`.\n\nPossible values are:\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "enum": [ + "ALL_SUCCESS", + "ALL_DONE", + "NONE_FAILED", + "AT_LEAST_ONE_SUCCESS", + "ALL_FAILED", + "AT_LEAST_ONE_FAILED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.RunJobTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "dbt_commands": { + "description": "An array of commands to execute for jobs with the dbt task, for example `\"dbt_commands\": [\"dbt deps\", \"dbt seed\", \"dbt deps\", \"dbt seed\", \"dbt run\"]`", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "jar_params": { + "description": "A list of parameters for jobs with Spark JAR tasks, for example `\"jar_params\": [\"john doe\", \"35\"]`.\nThe parameters are used to invoke the main function of the main class specified in the Spark JAR task.\nIf not specified upon `run-now`, it defaults to an empty list.\njar_params cannot be specified in conjunction with notebook_params.\nThe JSON representation of this field (for example `{\"jar_params\":[\"john doe\",\"35\"]}`) cannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "job_id": { + "description": "ID of the job to trigger.", + "$ref": "#/$defs/int64" + }, + "job_parameters": { + "description": "Job-level parameters used to trigger the job.", + "$ref": "#/$defs/map/string" + }, + "notebook_params": { + "description": "A map from keys to values for jobs with notebook task, for example `\"notebook_params\": {\"name\": \"john doe\", \"age\": \"35\"}`.\nThe map is passed to the notebook and is accessible through the [dbutils.widgets.get](https://docs.databricks.com/dev-tools/databricks-utils.html) function.\n\nIf not specified upon `run-now`, the triggered run uses the job’s base parameters.\n\nnotebook_params cannot be specified in conjunction with jar_params.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nThe JSON representation of this field (for example `{\"notebook_params\":{\"name\":\"john doe\",\"age\":\"35\"}}`) cannot exceed 10,000 bytes.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "pipeline_params": { + "description": "Controls whether the pipeline should perform a full refresh", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineParams" + }, + "python_named_params": { + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "python_params": { + "description": "A list of parameters for jobs with Python tasks, for example `\"python_params\": [\"john doe\", \"35\"]`.\nThe parameters are passed to Python file as command-line parameters. If specified upon `run-now`, it would overwrite\nthe parameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "spark_submit_params": { + "description": "A list of parameters for jobs with spark submit task, for example `\"spark_submit_params\": [\"--class\", \"org.apache.spark.examples.SparkPi\"]`.\nThe parameters are passed to spark-submit script as command-line parameters. If specified upon `run-now`, it would overwrite the\nparameters specified in job setting. The JSON representation of this field (for example `{\"python_params\":[\"john doe\",\"35\"]}`)\ncannot exceed 10,000 bytes.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs\n\nImportant\n\nThese parameters accept only Latin characters (ASCII character set). Using non-ASCII characters returns an error.\nExamples of invalid, non-ASCII characters are Chinese, Japanese kanjis, and emojis.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "sql_params": { + "description": "A map from keys to values for jobs with SQL task, for example `\"sql_params\": {\"name\": \"john doe\", \"age\": \"35\"}`. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + } + }, + "additionalProperties": false, + "required": [ + "job_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Source": { + "oneOf": [ + { + "type": "string", + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\\\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "enum": [ + "WORKSPACE", + "GIT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SparkJarTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "jar_uri": { + "description": "Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "main_class_name": { + "description": "The full name of the class containing the main method to be executed. This class must be contained in a JAR provided as a library.\n\nThe code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.", + "$ref": "#/$defs/string" + }, + "parameters": { + "description": "Parameters passed to the main method.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "run_as_repl": { + "description": "Deprecated. A value of `false` is no longer supported.", + "$ref": "#/$defs/bool", + "deprecationMessage": "This field is deprecated", + "deprecated": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SparkPythonTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command line parameters passed to the Python file.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + }, + "python_file": { + "description": "The Python file to be executed. Cloud file URIs (such as dbfs:/, s3:/, adls:/, gcs:/) and workspace paths are supported. For python files stored in the Databricks workspace, the path must be absolute and begin with `/`. For files stored in a remote repository, the path must be relative. This field is required.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the Python file. When set to `WORKSPACE` or not specified, the file will be retrieved from the local\nDatabricks workspace or cloud location (if the `python_file` has a URI format). When set to `GIT`,\nthe Python file will be retrieved from a Git repository defined in `git_source`.\n\n* `WORKSPACE`: The Python file is located in a Databricks workspace or at a cloud filesystem URI.\n* `GIT`: The Python file is located in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "python_file" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SparkSubmitTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "parameters": { + "description": "Command-line parameters passed to spark submit.\n\nUse [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTask": { + "oneOf": [ + { + "type": "object", + "properties": { + "alert": { + "description": "If alert, indicates that this job must refresh a SQL alert.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskAlert" + }, + "dashboard": { + "description": "If dashboard, indicates that this job must refresh a SQL dashboard.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskDashboard" + }, + "file": { + "description": "If file, indicates that this job runs a SQL file in a remote Git repository.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskFile" + }, + "parameters": { + "description": "Parameters to be used for each run of this job. The SQL alert task does not support custom parameters.", + "$ref": "#/$defs/map/string" + }, + "query": { + "description": "If query, indicates that this job must execute a SQL query.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskQuery" + }, + "warehouse_id": { + "description": "The canonical identifier of the SQL warehouse. Recommended to use with serverless or pro SQL warehouses. Classic SQL warehouses are only supported for SQL alert, dashboard and query tasks and are limited to scheduled single-task jobs.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "warehouse_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskAlert": { + "oneOf": [ + { + "type": "object", + "properties": { + "alert_id": { + "description": "The canonical identifier of the SQL alert.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the alert notifications are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, alert notifications are sent to subscribers.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "alert_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskDashboard": { + "oneOf": [ + { + "type": "object", + "properties": { + "custom_subject": { + "description": "Subject of the email sent to subscribers of this task.", + "$ref": "#/$defs/string" + }, + "dashboard_id": { + "description": "The canonical identifier of the SQL dashboard.", + "$ref": "#/$defs/string" + }, + "pause_subscriptions": { + "description": "If true, the dashboard snapshot is not taken, and emails are not sent to subscribers.", + "$ref": "#/$defs/bool" + }, + "subscriptions": { + "description": "If specified, dashboard snapshots are sent to subscriptions.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + "additionalProperties": false, + "required": [ + "dashboard_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskFile": { + "oneOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "Path of the SQL file. Must be relative if the source is a remote Git repository and absolute for workspace paths.", + "$ref": "#/$defs/string" + }, + "source": { + "description": "Optional location type of the SQL file. When set to `WORKSPACE`, the SQL file will be retrieved\nfrom the local Databricks workspace. When set to `GIT`, the SQL file will be retrieved from a Git repository\ndefined in `git_source`. If the value is empty, the task will use `GIT` if `git_source` is defined and `WORKSPACE` otherwise.\n\n* `WORKSPACE`: SQL file is located in Databricks workspace.\n* `GIT`: SQL file is located in cloud Git provider.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Source" + } + }, + "additionalProperties": false, + "required": [ + "path" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskQuery": { + "oneOf": [ + { + "type": "object", + "properties": { + "query_id": { + "description": "The canonical identifier of the SQL query.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "query_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination_id": { + "description": "The canonical identifier of the destination to receive email notification. This parameter is mutually exclusive with user_name. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The user name to receive the subscription email. This parameter is mutually exclusive with destination_id. You cannot set both destination_id and user_name for subscription notifications.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.StorageMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "DIRECT_QUERY", + "IMPORT", + "DUAL" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Subscription": { + "oneOf": [ + { + "type": "object", + "properties": { + "custom_subject": { + "description": "Optional: Allows users to specify a custom subject line on the email sent\nto subscribers.", + "$ref": "#/$defs/string" + }, + "paused": { + "description": "When true, the subscription will not send emails.", + "$ref": "#/$defs/bool" + }, + "subscribers": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.SubscriptionSubscriber" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SubscriptionSubscriber": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination_id": { + "$ref": "#/$defs/string" + }, + "user_name": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TableUpdateTriggerConfiguration": { + "oneOf": [ + { + "type": "object", + "properties": { + "condition": { + "description": "The table(s) condition based on which to trigger a job run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Condition" + }, + "min_time_between_triggers_seconds": { + "description": "If set, the trigger starts a run only after the specified amount of time has passed since\nthe last time the trigger fired. The minimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + }, + "table_names": { + "description": "A list of Delta tables to monitor for changes. The table name must be in the format `catalog_name.schema_name.table_name`.", + "$ref": "#/$defs/slice/string" + }, + "wait_after_last_change_seconds": { + "description": "If set, the trigger starts a run only after no table updates have occurred for the specified time\nand can be used to wait for a series of table updates before triggering a run. The\nminimum allowed value is 60 seconds.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Task": { + "oneOf": [ + { + "type": "object", + "properties": { + "clean_rooms_notebook_task": { + "description": "The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook\nwhen the `clean_rooms_notebook_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.CleanRoomsNotebookTask" + }, + "condition_task": { + "description": "The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present.\nThe condition task does not require a cluster to execute and does not support retries or notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ConditionTask" + }, + "dashboard_task": { + "description": "The task refreshes a dashboard and sends a snapshot to subscribers.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DashboardTask" + }, + "dbt_cloud_task": { + "description": "Task type for dbt cloud, deprecated in favor of the new name dbt_platform_task", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtCloudTask", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "dbt_platform_task": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtPlatformTask", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "dbt_task": { + "description": "The task runs one or more dbt commands when the `dbt_task` field is present. The dbt task requires both Databricks SQL and the ability to use a serverless or a pro SQL warehouse.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.DbtTask" + }, + "depends_on": { + "description": "An optional array of objects specifying the dependency graph of the task. All tasks specified in this field must complete before executing this task. The task will run only if the `run_if` condition is true.\nThe key is `task_key`, and the value is the name assigned to the dependent task.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + }, + "description": { + "description": "An optional description for this task.", + "$ref": "#/$defs/string" + }, + "disable_auto_optimization": { + "description": "An option to disable auto optimization in serverless", + "$ref": "#/$defs/bool" + }, + "email_notifications": { + "description": "An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications" + }, + "environment_key": { + "description": "The key that references an environment spec in a job. This field is required for Python script, Python wheel and dbt tasks when using serverless compute.", + "$ref": "#/$defs/string" + }, + "existing_cluster_id": { + "description": "If existing_cluster_id, the ID of an existing cluster that is used for all runs.\nWhen running jobs or tasks on an existing cluster, you may need to manually restart\nthe cluster if it stops responding. We suggest running jobs and tasks on new clusters for\ngreater reliability", + "$ref": "#/$defs/string" + }, + "for_each_task": { + "description": "The task executes a nested task for every input provided when the `for_each_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.ForEachTask" + }, + "gen_ai_compute_task": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.GenAiComputeTask", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "health": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRules" + }, + "job_cluster_key": { + "description": "If job_cluster_key, this task is executed reusing the cluster specified in `job.settings.job_clusters`.", + "$ref": "#/$defs/string" + }, + "libraries": { + "description": "An optional list of libraries to be installed on the cluster.\nThe default value is an empty list.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.Library" + }, + "max_retries": { + "description": "An optional maximum number of times to retry an unsuccessful run. A run is considered to be unsuccessful if it completes with the `FAILED` result_state or `INTERNAL_ERROR` `life_cycle_state`. The value `-1` means to retry indefinitely and the value `0` means to never retry.", + "$ref": "#/$defs/int" + }, + "min_retry_interval_millis": { + "description": "An optional minimal interval in milliseconds between the start of the failed run and the subsequent retry run. The default behavior is that unsuccessful runs are immediately retried.", + "$ref": "#/$defs/int" + }, + "new_cluster": { + "description": "If new_cluster, a description of a new cluster that is created for each run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterSpec" + }, + "notebook_task": { + "description": "The task runs a notebook when the `notebook_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.NotebookTask" + }, + "notification_settings": { + "description": "Optional notification settings that are used when sending notifications to each of the `email_notifications` and `webhook_notifications` for this task.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskNotificationSettings" + }, + "pipeline_task": { + "description": "The task triggers a pipeline update when the `pipeline_task` field is present. Only pipelines configured to use triggered more are supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PipelineTask" + }, + "power_bi_task": { + "description": "The task triggers a Power BI semantic model update when the `power_bi_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTask" + }, + "python_wheel_task": { + "description": "The task runs a Python wheel when the `python_wheel_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PythonWheelTask" + }, + "retry_on_timeout": { + "description": "An optional policy to specify whether to retry a job when it times out. The default behavior\nis to not retry on timeout.", + "$ref": "#/$defs/bool" + }, + "run_if": { + "description": "An optional value specifying the condition determining whether the task is run once its dependencies have been completed.\n\n* `ALL_SUCCESS`: All dependencies have executed and succeeded\n* `AT_LEAST_ONE_SUCCESS`: At least one dependency has succeeded\n* `NONE_FAILED`: None of the dependencies have failed and at least one was executed\n* `ALL_DONE`: All dependencies have been completed\n* `AT_LEAST_ONE_FAILED`: At least one dependency failed\n* `ALL_FAILED`: ALl dependencies have failed", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunIf" + }, + "run_job_task": { + "description": "The task triggers another job when the `run_job_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask" + }, + "spark_jar_task": { + "description": "The task runs a JAR when the `spark_jar_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask" + }, + "spark_python_task": { + "description": "The task runs a Python file when the `spark_python_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkPythonTask" + }, + "spark_submit_task": { + "description": "(Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.\n\nIn the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.\n\n`master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.\n\nBy default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.\n\nThe `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SparkSubmitTask" + }, + "sql_task": { + "description": "The task runs a SQL query or file, or it refreshes a SQL alert or a legacy SQL dashboard when the `sql_task` field is present.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTask" + }, + "task_key": { + "description": "A unique name for the task. This field is used to refer to this task from other tasks.\nThis field is required and must be unique within its parent job.\nOn Update or Reset, this field is used to reference the tasks to be updated or reset.", + "$ref": "#/$defs/string" + }, + "timeout_seconds": { + "description": "An optional timeout applied to each run of this job task. A value of `0` means no timeout.", + "$ref": "#/$defs/int" + }, + "webhook_notifications": { + "description": "A collection of system notification IDs to notify when runs of this task begin or complete. The default behavior is to not send any system notifications.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.WebhookNotifications" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskDependency": { + "oneOf": [ + { + "type": "object", + "properties": { + "outcome": { + "description": "Can only be specified on condition task dependencies. The outcome of the dependent task that must be met for this task to run.", + "$ref": "#/$defs/string" + }, + "task_key": { + "description": "The name of the task this task depends on.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "task_key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskEmailNotifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "no_alert_for_skipped_runs": { + "description": "If true, do not send email to recipients specified in `on_failure` if the run is skipped.\nThis field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.", + "$ref": "#/$defs/bool", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "on_duration_warning_threshold_exceeded": { + "description": "A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_failure": { + "description": "A list of email addresses to be notified when a run unsuccessfully completes. A run is considered to have completed unsuccessfully if it ends with an `INTERNAL_ERROR` `life_cycle_state` or a `FAILED`, or `TIMED_OUT` result_state. If this is not specified on job creation, reset, or update the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_start": { + "description": "A list of email addresses to be notified when a run begins. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + }, + "on_streaming_backlog_exceeded": { + "description": "A list of email addresses to notify when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.", + "$ref": "#/$defs/slice/string" + }, + "on_success": { + "description": "A list of email addresses to be notified when a run successfully completes. A run is considered to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS` result_state. If not specified on job creation, reset, or update, the list is empty, and notifications are not sent.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskNotificationSettings": { + "oneOf": [ + { + "type": "object", + "properties": { + "alert_on_last_attempt": { + "description": "If true, do not send notifications to recipients specified in `on_start` for the retried runs and do not send notifications to recipients specified in `on_failure` until the last retry of the run.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_canceled_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is canceled.", + "$ref": "#/$defs/bool" + }, + "no_alert_for_skipped_runs": { + "description": "If true, do not send notifications to recipients specified in `on_failure` if the run is skipped.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TriggerSettings": { + "oneOf": [ + { + "type": "object", + "properties": { + "file_arrival": { + "description": "File arrival trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.FileArrivalTriggerConfiguration" + }, + "pause_status": { + "description": "Whether this trigger is paused or not.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PauseStatus" + }, + "periodic": { + "description": "Periodic trigger settings.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PeriodicTriggerConfiguration" + }, + "table": { + "description": "Old table trigger settings name. Deprecated in favor of `table_update`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration", + "x-databricks-preview": "PRIVATE", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "table_update": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TableUpdateTriggerConfiguration", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Webhook": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.WebhookNotifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "on_duration_warning_threshold_exceeded": { + "description": "An optional list of system notification IDs to call when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. A maximum of 3 destinations can be specified for the `on_duration_warning_threshold_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_failure": { + "description": "An optional list of system notification IDs to call when the run fails. A maximum of 3 destinations can be specified for the `on_failure` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_start": { + "description": "An optional list of system notification IDs to call when the run starts. A maximum of 3 destinations can be specified for the `on_start` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_streaming_backlog_exceeded": { + "description": "An optional list of system notification IDs to call when any streaming backlog thresholds are exceeded for any stream.\nStreaming backlog thresholds can be set in the `health` field using the following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`.\nAlerting is based on the 10-minute average of these metrics. If the issue persists, notifications are resent every 30 minutes.\nA maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + }, + "on_success": { + "description": "An optional list of system notification IDs to call when the run completes successfully. A maximum of 3 destinations can be specified for the `on_success` property.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ExperimentTag": { + "oneOf": [ + { + "type": "object", + "description": "A tag for an experiment.", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelTag": { + "oneOf": [ + { + "type": "object", + "description": "Tag for a registered model", + "properties": { + "key": { + "description": "The tag key.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The tag value.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.CronTrigger": { + "oneOf": [ + { + "type": "object", + "properties": { + "quartz_cron_schedule": { + "$ref": "#/$defs/string" + }, + "timezone_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.DayOfWeek": { + "oneOf": [ + { + "type": "string", + "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.", + "enum": [ + "MONDAY", + "TUESDAY", + "WEDNESDAY", + "THURSDAY", + "FRIDAY", + "SATURDAY", + "SUNDAY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.DeploymentKind": { + "oneOf": [ + { + "type": "string", + "description": "The deployment method that manages the pipeline:\n- BUNDLE: The pipeline is managed by a Databricks Asset Bundle.", + "enum": [ + "BUNDLE" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.EventLogSpec": { + "oneOf": [ + { + "type": "object", + "description": "Configurable event log parameters.", + "properties": { + "catalog": { + "description": "The UC catalog the event log is published under.", + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name the event log is published to in UC.", + "$ref": "#/$defs/string" + }, + "schema": { + "description": "The UC schema the event log is published under.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.FileLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the source code.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Filters": { + "oneOf": [ + { + "type": "object", + "properties": { + "exclude": { + "description": "Paths to exclude.", + "$ref": "#/$defs/slice/string" + }, + "include": { + "description": "Paths to include.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "report": { + "description": "Select a specific source report.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ReportSpec" + }, + "schema": { + "description": "Select all tables from a specific source schema.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SchemaSpec" + }, + "table": { + "description": "Select a specific source table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpec" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionGatewayPipelineDefinition": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection_id": { + "description": "[Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "connection_name": { + "description": "Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.", + "$ref": "#/$defs/string" + }, + "gateway_storage_catalog": { + "description": "Required, Immutable. The name of the catalog for the gateway pipeline's storage location.", + "$ref": "#/$defs/string" + }, + "gateway_storage_name": { + "description": "Optional. The Unity Catalog-compatible name for the gateway storage location.\nThis is the destination to use for the data that is extracted by the gateway.\nDelta Live Tables system will automatically create the storage location under the catalog and schema.", + "$ref": "#/$defs/string" + }, + "gateway_storage_schema": { + "description": "Required, Immutable. The name of the schema for the gateway pipelines's storage location.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "connection_name", + "gateway_storage_catalog", + "gateway_storage_schema" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionPipelineDefinition": { + "oneOf": [ + { + "type": "object", + "properties": { + "connection_name": { + "description": "Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on.", + "$ref": "#/$defs/string" + }, + "ingestion_gateway_id": { + "description": "Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server.", + "$ref": "#/$defs/string" + }, + "objects": { + "description": "Required. Settings specifying tables to replicate and the destination for the replicated tables.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + }, + "source_type": { + "description": "The type of the foreign source.\nThe source type will be inferred from the source connection or ingestion gateway.\nThis field is output only and will be ignored if provided.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionSourceType" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig": { + "oneOf": [ + { + "type": "object", + "description": "Configurations that are only applicable for query-based ingestion connectors.", + "properties": { + "cursor_columns": { + "description": "The names of the monotonically increasing columns in the source table that are used to enable\nthe table to be read and ingested incrementally through structured streaming.\nThe columns are allowed to have repeated values but have to be non-decreasing.\nIf the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these\ncolumns will implicitly define the `sequence_by` behavior. You can still explicitly set\n`sequence_by` to override this default.", + "$ref": "#/$defs/slice/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "deletion_condition": { + "description": "Specifies a SQL WHERE condition that specifies that the source row has been deleted.\nThis is sometimes referred to as \"soft-deletes\".\nFor example: \"Operation = 'DELETE'\" or \"is_deleted = true\".\nThis field is orthogonal to `hard_deletion_sync_interval_in_seconds`,\none for soft-deletes and the other for hard-deletes.\nSee also the hard_deletion_sync_min_interval_in_seconds field for\nhandling of \"hard deletes\" where the source rows are physically removed from the table.", + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "hard_deletion_sync_min_interval_in_seconds": { + "description": "Specifies the minimum interval (in seconds) between snapshots on primary keys\nfor detecting and synchronizing hard deletions—i.e., rows that have been\nphysically removed from the source table.\nThis interval acts as a lower bound. If ingestion runs less frequently than\nthis value, hard deletion synchronization will align with the actual ingestion\nfrequency instead of happening more often.\nIf not set, hard deletion synchronization via snapshots is disabled.\nThis field is mutable and can be updated without triggering a full snapshot.", + "$ref": "#/$defs/int64", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionSourceType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "MYSQL", + "POSTGRESQL", + "REDSHIFT", + "SQLDW", + "SQLSERVER", + "SALESFORCE", + "BIGQUERY", + "NETSUITE", + "WORKDAY_RAAS", + "GA4_RAW_DATA", + "SERVICENOW", + "MANAGED_POSTGRESQL", + "ORACLE", + "TERADATA", + "SHAREPOINT", + "DYNAMICS365", + "CONFLUENCE", + "META_MARKETING" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.ManualTrigger": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.NotebookLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "path": { + "description": "The absolute path of the source code.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Notifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "alerts": { + "description": "A list of alerts that trigger the sending of notifications to the configured\ndestinations. The supported alerts are:\n\n* `on-update-success`: A pipeline update completes successfully.\n* `on-update-failure`: Each time a pipeline update fails.\n* `on-update-fatal-failure`: A pipeline update fails with a non-retryable (fatal) error.\n* `on-flow-failure`: A single data flow fails.", + "$ref": "#/$defs/slice/string" + }, + "email_recipients": { + "description": "A list of email addresses notified when a configured alert is triggered.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PathPattern": { + "oneOf": [ + { + "type": "object", + "properties": { + "include": { + "description": "The source code to include for pipelines", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "oneOf": [ + { + "type": "object", + "properties": { + "apply_policy_default_values": { + "description": "Note: This field won't be persisted. Only API users will check this field.", + "$ref": "#/$defs/bool" + }, + "autoscale": { + "description": "Parameters needed in order to automatically scale clusters up and down based on load.\nNote: autoscaling works best with DB runtime versions 3.0 or later.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscale" + }, + "aws_attributes": { + "description": "Attributes related to clusters running on Amazon Web Services.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AwsAttributes" + }, + "azure_attributes": { + "description": "Attributes related to clusters running on Microsoft Azure.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.AzureAttributes" + }, + "cluster_log_conf": { + "description": "The configuration for delivering spark logs to a long-term storage destination.\nOnly dbfs destinations are supported. Only one destination can be specified\nfor one cluster. If the conf is given, the logs will be delivered to the destination every\n`5 mins`. The destination of driver logs is `$destination/$clusterId/driver`, while\nthe destination of executor logs is `$destination/$clusterId/executor`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ClusterLogConf" + }, + "custom_tags": { + "description": "Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS\ninstances and EBS volumes) with these tags in addition to `default_tags`. Notes:\n\n- Currently, Databricks allows at most 45 custom tags\n\n- Clusters can only reuse cloud resources if the resources' tags are a subset of the cluster tags", + "$ref": "#/$defs/map/string" + }, + "driver_instance_pool_id": { + "description": "The optional ID of the instance pool for the driver of the cluster belongs.\nThe pool cluster uses the instance pool with id (instance_pool_id) if the driver pool is not\nassigned.", + "$ref": "#/$defs/string" + }, + "driver_node_type_id": { + "description": "The node type of the Spark driver.\nNote that this field is optional; if unset, the driver node type will be set as the same value\nas `node_type_id` defined above.", + "$ref": "#/$defs/string" + }, + "enable_local_disk_encryption": { + "description": "Whether to enable local disk encryption for the cluster.", + "$ref": "#/$defs/bool" + }, + "gcp_attributes": { + "description": "Attributes related to clusters running on Google Cloud Platform.\nIf not specified at cluster creation, a set of default values will be used.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes" + }, + "init_scripts": { + "description": "The configuration for storing init scripts. Any number of destinations can be specified. The scripts are executed sequentially in the order provided. If `cluster_log_conf` is specified, init script logs are sent to `\u003cdestination\u003e/\u003ccluster-ID\u003e/init_scripts`.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + }, + "instance_pool_id": { + "description": "The optional ID of the instance pool to which the cluster belongs.", + "$ref": "#/$defs/string" + }, + "label": { + "description": "A label for the cluster specification, either `default` to configure the default cluster, or `maintenance` to configure the maintenance cluster. This field is optional. The default value is `default`.", + "$ref": "#/$defs/string" + }, + "node_type_id": { + "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", + "$ref": "#/$defs/string" + }, + "num_workers": { + "description": "Number of worker nodes that this cluster should have. A cluster has one Spark Driver\nand `num_workers` Executors for a total of `num_workers` + 1 Spark nodes.\n\nNote: When reading the properties of a cluster, this field reflects the desired number\nof workers rather than the actual current number of workers. For instance, if a cluster\nis resized from 5 to 10 workers, this field will immediately be updated to reflect\nthe target size of 10 workers, whereas the workers listed in `spark_info` will gradually\nincrease from 5 to 10 as the new nodes are provisioned.", + "$ref": "#/$defs/int" + }, + "policy_id": { + "description": "The ID of the cluster policy used to create the cluster if applicable.", + "$ref": "#/$defs/string" + }, + "spark_conf": { + "description": "An object containing a set of optional, user-specified Spark configuration key-value pairs.\nSee :method:clusters/create for more details.", + "$ref": "#/$defs/map/string" + }, + "spark_env_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs.\nPlease note that key-value pair of the form (X,Y) will be exported as is (i.e.,\n`export X='Y'`) while launching the driver and workers.\n\nIn order to specify an additional set of `SPARK_DAEMON_JAVA_OPTS`, we recommend appending\nthem to `$SPARK_DAEMON_JAVA_OPTS` as shown in the example below. This ensures that all\ndefault databricks managed environmental variables are included as well.\n\nExample Spark environment variables:\n`{\"SPARK_WORKER_MEMORY\": \"28000m\", \"SPARK_LOCAL_DIRS\": \"/local_disk0\"}` or\n`{\"SPARK_DAEMON_JAVA_OPTS\": \"$SPARK_DAEMON_JAVA_OPTS -Dspark.shuffle.service.enabled=true\"}`", + "$ref": "#/$defs/map/string" + }, + "ssh_public_keys": { + "description": "SSH public key contents that will be added to each Spark node in this cluster. The\ncorresponding private keys can be used to login with the user name `ubuntu` on port `2200`.\nUp to 10 keys can be specified.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscale": { + "oneOf": [ + { + "type": "object", + "properties": { + "max_workers": { + "description": "The maximum number of workers to which the cluster can scale up when overloaded. `max_workers` must be strictly greater than `min_workers`.", + "$ref": "#/$defs/int" + }, + "min_workers": { + "description": "The minimum number of workers the cluster can scale down to when underutilized.\nIt is also the initial number of workers the cluster will have after creation.", + "$ref": "#/$defs/int" + }, + "mode": { + "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineClusterAutoscaleMode" + } + }, + "additionalProperties": false, + "required": [ + "max_workers", + "min_workers" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineClusterAutoscaleMode": { + "oneOf": [ + { + "type": "string", + "description": "Databricks Enhanced Autoscaling optimizes cluster utilization by automatically\nallocating cluster resources based on workload volume, with minimal impact to\nthe data processing latency of your pipelines. Enhanced Autoscaling is available\nfor `updates` clusters only. The legacy autoscaling feature is used for `maintenance`\nclusters.", + "enum": [ + "ENHANCED", + "LEGACY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineDeployment": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "description": "The deployment method that manages the pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DeploymentKind" + }, + "metadata_file_path": { + "description": "The path to the file containing metadata about the deployment.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "kind" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "oneOf": [ + { + "type": "object", + "properties": { + "file": { + "description": "The path to a file that defines a pipeline and is stored in the Databricks Repos.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.FileLibrary" + }, + "glob": { + "description": "The unified field to include source codes.\nEach entry can be a notebook path, a file path, or a folder path that ends `/**`.\nThis field cannot be used together with `notebook` or `file`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PathPattern" + }, + "jar": { + "description": "URI of the jar to be installed. Currently only DBFS is supported.", + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "maven": { + "description": "Specification of a maven library to be installed.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.MavenLibrary", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "notebook": { + "description": "The path to a notebook that defines a pipeline and is stored in the Databricks workspace.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary" + }, + "whl": { + "description": "URI of the whl to be installed.", + "$ref": "#/$defs/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineTrigger": { + "oneOf": [ + { + "type": "object", + "properties": { + "cron": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger" + }, + "manual": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelinesEnvironment": { + "oneOf": [ + { + "type": "object", + "description": "The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines.\nIn this minimal environment spec, only pip dependencies are supported.", + "properties": { + "dependencies": { + "description": "List of pip dependencies, as supported by the version of pip in this environment.\nEach dependency is a pip requirement file line https://pip.pypa.io/en/stable/reference/requirements-file-format/\nAllowed dependency could be \u003crequirement specifier\u003e, \u003carchive url/path\u003e, \u003clocal project path\u003e(WSFS or Volumes in Databricks), \u003cvcs project url\u003e", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.ReportSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store table.", + "$ref": "#/$defs/string" + }, + "destination_table": { + "description": "Required. Destination table name. The pipeline fails if a table with that name already exists.", + "$ref": "#/$defs/string" + }, + "source_url": { + "description": "Required. Report URL in the source system.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false, + "required": [ + "destination_catalog", + "destination_schema", + "source_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.RestartWindow": { + "oneOf": [ + { + "type": "object", + "properties": { + "days_of_week": { + "description": "Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour).\nIf not specified all days of the week will be used.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/pipelines.DayOfWeek" + }, + "start_hour": { + "description": "An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day.\nContinuous pipeline restart is triggered only within a five-hour window starting at this hour.", + "$ref": "#/$defs/int" + }, + "time_zone_id": { + "description": "Time zone id of restart window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.\nIf not specified, UTC will be used.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "start_hour" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.RunAs": { + "oneOf": [ + { + "type": "object", + "description": "Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.\n\nOnly `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.", + "properties": { + "service_principal_name": { + "description": "Application ID of an active service principal. Setting this field requires the `servicePrincipal/user` role.", + "$ref": "#/$defs/string" + }, + "user_name": { + "description": "The email of an active workspace user. Users can only set this field to their own email.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.SchemaSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store tables.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store tables in. Tables with the same name as the source tables are created in this destination schema. The pipeline fails If a table with the same name already exists.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "The source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Required. Schema name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings are applied to all tables in this schema and override the table_configuration defined in the IngestionPipelineDefinition object.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false, + "required": [ + "destination_catalog", + "destination_schema", + "source_schema" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "destination_catalog": { + "description": "Required. Destination catalog to store table.", + "$ref": "#/$defs/string" + }, + "destination_schema": { + "description": "Required. Destination schema to store table.", + "$ref": "#/$defs/string" + }, + "destination_table": { + "description": "Optional. Destination table name. The pipeline fails if a table with that name already exists. If not set, the source table name is used.", + "$ref": "#/$defs/string" + }, + "source_catalog": { + "description": "Source catalog name. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_schema": { + "description": "Schema name in the source database. Might be optional depending on the type of source.", + "$ref": "#/$defs/string" + }, + "source_table": { + "description": "Required. Table name in the source database.", + "$ref": "#/$defs/string" + }, + "table_configuration": { + "description": "Configuration settings to control the ingestion of tables. These settings override the table_configuration defined in the IngestionPipelineDefinition object and the SchemaSpec.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfig" + } + }, + "additionalProperties": false, + "required": [ + "destination_catalog", + "destination_schema", + "source_table" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpecificConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "exclude_columns": { + "description": "A list of column names to be excluded for the ingestion.\nWhen not specified, include_columns fully controls what columns to be ingested.\nWhen specified, all other columns including future ones will be automatically included for ingestion.\nThis field in mutually exclusive with `include_columns`.", + "$ref": "#/$defs/slice/string" + }, + "include_columns": { + "description": "A list of column names to be included for the ingestion.\nWhen not specified, all columns except ones in exclude_columns will be included. Future\ncolumns will be automatically included.\nWhen specified, all other future columns will be automatically excluded from ingestion.\nThis field in mutually exclusive with `exclude_columns`.", + "$ref": "#/$defs/slice/string" + }, + "primary_keys": { + "description": "The primary key of the table used to apply changes.", + "$ref": "#/$defs/slice/string" + }, + "query_based_connector_config": { + "description": "Configurations that are only applicable for query-based ingestion connectors.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "salesforce_include_formula_fields": { + "description": "If true, formula fields defined in the table are included in the ingestion. This setting is only valid for the Salesforce connector", + "$ref": "#/$defs/bool", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "scd_type": { + "description": "The SCD type to use to ingest the table.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScdType", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "sequence_by": { + "description": "The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.TableSpecificConfigScdType": { + "oneOf": [ + { + "type": "string", + "description": "The SCD type to use to ingest the table.", + "enum": [ + "SCD_TYPE_1", + "SCD_TYPE_2", + "APPEND_ONLY" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.Ai21LabsConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "ai21labs_api_key": { + "description": "The Databricks secret key reference for an AI21 Labs API key. If you\nprefer to paste your API key directly, see `ai21labs_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "ai21labs_api_key_plaintext": { + "description": "An AI21 Labs API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `ai21labs_api_key`. You\nmust provide an API key using one of the following fields:\n`ai21labs_api_key` or `ai21labs_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "fallback_config": { + "description": "Configuration for traffic fallback which auto fallbacks to other served entities if the request to a served\nentity fails with certain error codes, to increase availability.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.FallbackConfig" + }, + "guardrails": { + "description": "Configuration for AI Guardrails to prevent unwanted data and unsafe data in requests and responses.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrails" + }, + "inference_table_config": { + "description": "Configuration for payload logging using inference tables.\nUse these tables to monitor and audit data being sent to and received from model APIs and to improve model quality.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayInferenceTableConfig" + }, + "rate_limits": { + "description": "Configuration for rate limits which can be set to limit endpoint traffic.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + }, + "usage_tracking_config": { + "description": "Configuration to enable usage tracking using system tables.\nThese tables allow you to monitor operational usage on endpoints and their associated costs.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayUsageTrackingConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailParameters": { + "oneOf": [ + { + "type": "object", + "properties": { + "invalid_keywords": { + "description": "List of invalid keywords.\nAI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.", + "$ref": "#/$defs/slice/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + }, + "pii": { + "description": "Configuration for guardrail PII filter.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior" + }, + "safety": { + "description": "Indicates whether the safety filter is enabled.", + "$ref": "#/$defs/bool" + }, + "valid_topics": { + "description": "The list of allowed topics.\nGiven a chat request, this guardrail flags the request if its topic is not in the allowed topics.", + "$ref": "#/$defs/slice/string", + "deprecationMessage": "This field is deprecated", + "deprecated": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehavior": { + "oneOf": [ + { + "type": "object", + "properties": { + "behavior": { + "description": "Configuration for input guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehaviorBehavior" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrailPiiBehaviorBehavior": { + "oneOf": [ + { + "type": "string", + "enum": [ + "NONE", + "BLOCK", + "MASK" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayGuardrails": { + "oneOf": [ + { + "type": "object", + "properties": { + "input": { + "description": "Configuration for input guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + }, + "output": { + "description": "Configuration for output guardrail filters.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayInferenceTableConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the catalog name.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. Required when enabling inference tables.\nNOTE: On update, you have to disable inference table first in order to change the schema name.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog.\nNOTE: On update, you have to disable inference table first in order to change the prefix name.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimit": { + "oneOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int64" + }, + "key": { + "description": "Key field for a rate limit. Currently, 'user', 'user_group, 'service_principal', and 'endpoint' are supported,\nwith 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey" + }, + "principal": { + "description": "Principal field for a user, user group, or service principal to apply rate limiting to. Accepts a user email, group name, or service principal application ID.", + "$ref": "#/$defs/string" + }, + "renewal_period": { + "description": "Renewal period field for a rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod" + }, + "tokens": { + "description": "Used to specify how many tokens are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false, + "required": [ + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimitKey": { + "oneOf": [ + { + "type": "string", + "enum": [ + "user", + "endpoint", + "user_group", + "service_principal" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimitRenewalPeriod": { + "oneOf": [ + { + "type": "string", + "enum": [ + "minute" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayUsageTrackingConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to enable usage tracking.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AmazonBedrockConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "aws_access_key_id": { + "description": "The Databricks secret key reference for an AWS access key ID with\npermissions to interact with Bedrock services. If you prefer to paste\nyour API key directly, see `aws_access_key_id_plaintext`. You must provide an API\nkey using one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_access_key_id_plaintext": { + "description": "An AWS access key ID with permissions to interact with Bedrock services\nprovided as a plaintext string. If you prefer to reference your key using\nDatabricks Secrets, see `aws_access_key_id`. You must provide an API key\nusing one of the following fields: `aws_access_key_id` or\n`aws_access_key_id_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_region": { + "description": "The AWS region to use. Bedrock has to be enabled there.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key": { + "description": "The Databricks secret key reference for an AWS secret access key paired\nwith the access key ID, with permissions to interact with Bedrock\nservices. If you prefer to paste your API key directly, see\n`aws_secret_access_key_plaintext`. You must provide an API key using one\nof the following fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "aws_secret_access_key_plaintext": { + "description": "An AWS secret access key paired with the access key ID, with permissions\nto interact with Bedrock services provided as a plaintext string. If you\nprefer to reference your key using Databricks Secrets, see\n`aws_secret_access_key`. You must provide an API key using one of the\nfollowing fields: `aws_secret_access_key` or\n`aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "bedrock_provider": { + "description": "The underlying provider in Amazon Bedrock. Supported values (case\ninsensitive) include: Anthropic, Cohere, AI21Labs, Amazon.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfigBedrockProvider" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the external model will use to access AWS resources.\nYou must authenticate using an instance profile or access keys.\nIf you prefer to authenticate using access keys, see `aws_access_key_id`,\n`aws_access_key_id_plaintext`, `aws_secret_access_key` and `aws_secret_access_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "aws_region", + "bedrock_provider" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AmazonBedrockConfigBedrockProvider": { + "oneOf": [ + { + "type": "string", + "enum": [ + "anthropic", + "cohere", + "ai21labs", + "amazon" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AnthropicConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "anthropic_api_key": { + "description": "The Databricks secret key reference for an Anthropic API key. If you\nprefer to paste your API key directly, see `anthropic_api_key_plaintext`.\nYou must provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "anthropic_api_key_plaintext": { + "description": "The Anthropic API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `anthropic_api_key`. You\nmust provide an API key using one of the following fields:\n`anthropic_api_key` or `anthropic_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ApiKeyAuth": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "The name of the API key parameter used for authentication.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The Databricks secret key reference for an API Key.\nIf you prefer to paste your token directly, see `value_plaintext`.", + "$ref": "#/$defs/string" + }, + "value_plaintext": { + "description": "The API Key provided as a plaintext string. If you prefer to reference your\ntoken using Databricks Secrets, see `value`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AutoCaptureConfigInput": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "enabled": { + "description": "Indicates whether the inference table is enabled.", + "$ref": "#/$defs/bool" + }, + "schema_name": { + "description": "The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled.", + "$ref": "#/$defs/string" + }, + "table_name_prefix": { + "description": "The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.BearerTokenAuth": { + "oneOf": [ + { + "type": "object", + "properties": { + "token": { + "description": "The Databricks secret key reference for a token.\nIf you prefer to paste your token directly, see `token_plaintext`.", + "$ref": "#/$defs/string" + }, + "token_plaintext": { + "description": "The token provided as a plaintext string. If you prefer to reference your\ntoken using Databricks Secrets, see `token`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.CohereConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "cohere_api_base": { + "description": "This is an optional field to provide a customized base URL for the Cohere\nAPI. If left unspecified, the standard Cohere base URL is used.", + "$ref": "#/$defs/string" + }, + "cohere_api_key": { + "description": "The Databricks secret key reference for a Cohere API key. If you prefer\nto paste your API key directly, see `cohere_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `cohere_api_key` or\n`cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "cohere_api_key_plaintext": { + "description": "The Cohere API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `cohere_api_key`. You\nmust provide an API key using one of the following fields:\n`cohere_api_key` or `cohere_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.CustomProviderConfig": { + "oneOf": [ + { + "type": "object", + "description": "Configs needed to create a custom provider model route.", + "properties": { + "api_key_auth": { + "description": "This is a field to provide API key authentication for the custom provider API.\nYou can only specify one authentication method.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ApiKeyAuth" + }, + "bearer_token_auth": { + "description": "This is a field to provide bearer token authentication for the custom provider API.\nYou can only specify one authentication method.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.BearerTokenAuth" + }, + "custom_provider_url": { + "description": "This is a field to provide the URL of the custom provider API.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "custom_provider_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.DatabricksModelServingConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "databricks_api_token": { + "description": "The Databricks secret key reference for a Databricks API token that\ncorresponds to a user or service principal with Can Query access to the\nmodel serving endpoint pointed to by this external model. If you prefer\nto paste your API key directly, see `databricks_api_token_plaintext`. You\nmust provide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.", + "$ref": "#/$defs/string" + }, + "databricks_api_token_plaintext": { + "description": "The Databricks API token that corresponds to a user or service principal\nwith Can Query access to the model serving endpoint pointed to by this\nexternal model provided as a plaintext string. If you prefer to reference\nyour key using Databricks Secrets, see `databricks_api_token`. You must\nprovide an API key using one of the following fields:\n`databricks_api_token` or `databricks_api_token_plaintext`.", + "$ref": "#/$defs/string" + }, + "databricks_workspace_url": { + "description": "The URL of the Databricks workspace containing the model serving endpoint\npointed to by this external model.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "databricks_workspace_url" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EmailNotifications": { + "oneOf": [ + { + "type": "object", + "properties": { + "on_update_failure": { + "description": "A list of email addresses to be notified when an endpoint fails to update its configuration or state.", + "$ref": "#/$defs/slice/string" + }, + "on_update_success": { + "description": "A list of email addresses to be notified when an endpoint successfully updates its configuration or state.", + "$ref": "#/$defs/slice/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointCoreConfigInput": { + "oneOf": [ + { + "type": "object", + "properties": { + "auto_capture_config": { + "description": "Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog.\nNote: this field is deprecated for creating new provisioned throughput endpoints,\nor updating existing provisioned throughput endpoints that never have inference table configured;\nin these cases please use AI Gateway to manage inference tables.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AutoCaptureConfigInput" + }, + "served_entities": { + "description": "The list of served entities under the serving endpoint config.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + }, + "served_models": { + "description": "(Deprecated, use served_entities instead) The list of served models under the serving endpoint config.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + }, + "traffic_config": { + "description": "The traffic configuration associated with the serving endpoint config.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointTag": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "description": "Key field for a serving endpoint tag.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "Optional value field for a serving endpoint tag.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "key" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ExternalModel": { + "oneOf": [ + { + "type": "object", + "properties": { + "ai21labs_config": { + "description": "AI21Labs Config. Only required if the provider is 'ai21labs'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig" + }, + "amazon_bedrock_config": { + "description": "Amazon Bedrock Config. Only required if the provider is 'amazon-bedrock'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AmazonBedrockConfig" + }, + "anthropic_config": { + "description": "Anthropic Config. Only required if the provider is 'anthropic'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AnthropicConfig" + }, + "cohere_config": { + "description": "Cohere Config. Only required if the provider is 'cohere'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.CohereConfig" + }, + "custom_provider_config": { + "description": "Custom Provider Config. Only required if the provider is 'custom'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.CustomProviderConfig" + }, + "databricks_model_serving_config": { + "description": "Databricks Model Serving Config. Only required if the provider is 'databricks-model-serving'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.DatabricksModelServingConfig" + }, + "google_cloud_vertex_ai_config": { + "description": "Google Cloud Vertex AI Config. Only required if the provider is 'google-cloud-vertex-ai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.GoogleCloudVertexAiConfig" + }, + "name": { + "description": "The name of the external model.", + "$ref": "#/$defs/string" + }, + "openai_config": { + "description": "OpenAI Config. Only required if the provider is 'openai'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.OpenAiConfig" + }, + "palm_config": { + "description": "PaLM Config. Only required if the provider is 'palm'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.PaLmConfig" + }, + "provider": { + "description": "The name of the provider for the external model. Currently, the supported providers are 'ai21labs', 'anthropic', 'amazon-bedrock', 'cohere', 'databricks-model-serving', 'google-cloud-vertex-ai', 'openai', 'palm', and 'custom'.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModelProvider" + }, + "task": { + "description": "The task type of the external model.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "provider", + "task" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ExternalModelProvider": { + "oneOf": [ + { + "type": "string", + "enum": [ + "ai21labs", + "anthropic", + "amazon-bedrock", + "cohere", + "databricks-model-serving", + "google-cloud-vertex-ai", + "openai", + "palm", + "custom" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.FallbackConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to enable traffic fallback. When a served entity in the serving endpoint returns specific error\ncodes (e.g. 500), the request will automatically be round-robin attempted with other served entities in the same\nendpoint, following the order of served entity list, until a successful response is returned.\nIf all attempts fail, return the last response with the error code.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false, + "required": [ + "enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.GoogleCloudVertexAiConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "private_key": { + "description": "The Databricks secret key reference for a private key for the service\naccount which has access to the Google Cloud Vertex AI Service. See [Best\npractices for managing service account keys]. If you prefer to paste your\nAPI key directly, see `private_key_plaintext`. You must provide an API\nkey using one of the following fields: `private_key` or\n`private_key_plaintext`\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys", + "$ref": "#/$defs/string" + }, + "private_key_plaintext": { + "description": "The private key for the service account which has access to the Google\nCloud Vertex AI Service provided as a plaintext secret. See [Best\npractices for managing service account keys]. If you prefer to reference\nyour key using Databricks Secrets, see `private_key`. You must provide an\nAPI key using one of the following fields: `private_key` or\n`private_key_plaintext`.\n\n[Best practices for managing service account keys]: https://cloud.google.com/iam/docs/best-practices-for-managing-service-account-keys", + "$ref": "#/$defs/string" + }, + "project_id": { + "description": "This is the Google Cloud project id that the service account is\nassociated with.", + "$ref": "#/$defs/string" + }, + "region": { + "description": "This is the region for the Google Cloud Vertex AI Service. See [supported\nregions] for more details. Some models are only available in specific\nregions.\n\n[supported regions]: https://cloud.google.com/vertex-ai/docs/general/locations", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "project_id", + "region" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.OpenAiConfig": { + "oneOf": [ + { + "type": "object", + "description": "Configs needed to create an OpenAI model route.", + "properties": { + "microsoft_entra_client_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Client ID.", + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret": { + "description": "The Databricks secret key reference for a client secret used for\nMicrosoft Entra ID authentication. If you prefer to paste your client\nsecret directly, see `microsoft_entra_client_secret_plaintext`. You must\nprovide an API key using one of the following fields:\n`microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.", + "$ref": "#/$defs/string" + }, + "microsoft_entra_client_secret_plaintext": { + "description": "The client secret used for Microsoft Entra ID authentication provided as\na plaintext string. If you prefer to reference your key using Databricks\nSecrets, see `microsoft_entra_client_secret`. You must provide an API key\nusing one of the following fields: `microsoft_entra_client_secret` or\n`microsoft_entra_client_secret_plaintext`.", + "$ref": "#/$defs/string" + }, + "microsoft_entra_tenant_id": { + "description": "This field is only required for Azure AD OpenAI and is the Microsoft\nEntra Tenant ID.", + "$ref": "#/$defs/string" + }, + "openai_api_base": { + "description": "This is a field to provide a customized base URl for the OpenAI API. For\nAzure OpenAI, this field is required, and is the base URL for the Azure\nOpenAI API service provided by Azure. For other OpenAI API types, this\nfield is optional, and if left unspecified, the standard OpenAI base URL\nis used.", + "$ref": "#/$defs/string" + }, + "openai_api_key": { + "description": "The Databricks secret key reference for an OpenAI API key using the\nOpenAI or Azure service. If you prefer to paste your API key directly,\nsee `openai_api_key_plaintext`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "openai_api_key_plaintext": { + "description": "The OpenAI API key using the OpenAI or Azure service provided as a\nplaintext string. If you prefer to reference your key using Databricks\nSecrets, see `openai_api_key`. You must provide an API key using one of\nthe following fields: `openai_api_key` or `openai_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "openai_api_type": { + "description": "This is an optional field to specify the type of OpenAI API to use. For\nAzure OpenAI, this field is required, and adjust this parameter to\nrepresent the preferred security access validation protocol. For access\ntoken validation, use azure. For authentication using Azure Active\nDirectory (Azure AD) use, azuread.", + "$ref": "#/$defs/string" + }, + "openai_api_version": { + "description": "This is an optional field to specify the OpenAI API version. For Azure\nOpenAI, this field is required, and is the version of the Azure OpenAI\nservice to utilize, specified by a date.", + "$ref": "#/$defs/string" + }, + "openai_deployment_name": { + "description": "This field is only required for Azure OpenAI and is the name of the\ndeployment resource for the Azure OpenAI service.", + "$ref": "#/$defs/string" + }, + "openai_organization": { + "description": "This is an optional field to specify the organization in OpenAI or Azure\nOpenAI.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.PaLmConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "palm_api_key": { + "description": "The Databricks secret key reference for a PaLM API key. If you prefer to\npaste your API key directly, see `palm_api_key_plaintext`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.", + "$ref": "#/$defs/string" + }, + "palm_api_key_plaintext": { + "description": "The PaLM API key provided as a plaintext string. If you prefer to\nreference your key using Databricks Secrets, see `palm_api_key`. You must\nprovide an API key using one of the following fields: `palm_api_key` or\n`palm_api_key_plaintext`.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimit": { + "oneOf": [ + { + "type": "object", + "properties": { + "calls": { + "description": "Used to specify how many calls are allowed for a key within the renewal_period.", + "$ref": "#/$defs/int64" + }, + "key": { + "description": "Key field for a serving endpoint rate limit. Currently, only 'user' and 'endpoint' are supported, with 'endpoint' being the default if not specified.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitKey" + }, + "renewal_period": { + "description": "Renewal period field for a serving endpoint rate limit. Currently, only 'minute' is supported.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod" + } + }, + "additionalProperties": false, + "required": [ + "calls", + "renewal_period" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimitKey": { + "oneOf": [ + { + "type": "string", + "enum": [ + "user", + "endpoint" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimitRenewalPeriod": { + "oneOf": [ + { + "type": "string", + "enum": [ + "minute" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.Route": { + "oneOf": [ + { + "type": "object", + "properties": { + "served_entity_name": { + "$ref": "#/$defs/string" + }, + "served_model_name": { + "description": "The name of the served model this route configures traffic for.", + "$ref": "#/$defs/string" + }, + "traffic_percentage": { + "description": "The percentage of endpoint traffic to send to this route. It must be an integer between 0 and 100 inclusive.", + "$ref": "#/$defs/int" + } + }, + "additionalProperties": false, + "required": [ + "traffic_percentage" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "oneOf": [ + { + "type": "object", + "properties": { + "entity_name": { + "description": "The name of the entity to be served. The entity may be a model in the Databricks Model Registry, a model in the Unity Catalog (UC), or a function of type FEATURE_SPEC in the UC. If it is a UC object, the full name of the object should be given in the form of **catalog_name.schema_name.model_name**.", + "$ref": "#/$defs/string" + }, + "entity_version": { + "$ref": "#/$defs/string" + }, + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "external_model": { + "description": "The external model to be served. NOTE: Only one of external_model and (entity_name, entity_version, workload_size, workload_type, and scale_to_zero_enabled) can be specified with the latter set being used for custom model serving for a Databricks registered model. For an existing endpoint with external_model, it cannot be updated to an endpoint without external_model. If the endpoint is created without external_model, users cannot update it to add external_model later. The task type of all external models within an endpoint must be the same.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ExternalModel" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", + "$ref": "#/$defs/string" + }, + "max_provisioned_concurrency": { + "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_concurrency": { + "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, + "name": { + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", + "$ref": "#/$defs/string" + }, + "provisioned_model_units": { + "description": "The number of model units provisioned.", + "$ref": "#/$defs/int64" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served entity should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", + "$ref": "#/$defs/string" + }, + "workload_type": { + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServingModelWorkloadType" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInput": { + "oneOf": [ + { + "type": "object", + "properties": { + "environment_vars": { + "description": "An object containing a set of optional, user-specified environment variable key-value pairs used for serving this entity. Note: this is an experimental feature and subject to change. Example entity environment variables that refer to Databricks secrets: `{\"OPENAI_API_KEY\": \"{{secrets/my_scope/my_key}}\", \"DATABRICKS_TOKEN\": \"{{secrets/my_scope2/my_key2}}\"}`", + "$ref": "#/$defs/map/string" + }, + "instance_profile_arn": { + "description": "ARN of the instance profile that the served entity uses to access AWS resources.", + "$ref": "#/$defs/string" + }, + "max_provisioned_concurrency": { + "description": "The maximum provisioned concurrency that the endpoint can scale up to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, + "max_provisioned_throughput": { + "description": "The maximum tokens per second that the endpoint can scale up to.", + "$ref": "#/$defs/int" + }, + "min_provisioned_concurrency": { + "description": "The minimum provisioned concurrency that the endpoint can scale down to. Do not use if workload_size is specified.", + "$ref": "#/$defs/int" + }, + "min_provisioned_throughput": { + "description": "The minimum tokens per second that the endpoint can scale down to.", + "$ref": "#/$defs/int" + }, + "model_name": { + "$ref": "#/$defs/string" + }, + "model_version": { + "$ref": "#/$defs/string" + }, + "name": { + "description": "The name of a served entity. It must be unique across an endpoint. A served entity name can consist of alphanumeric characters, dashes, and underscores. If not specified for an external model, this field defaults to external_model.name, with '.' and ':' replaced with '-', and if not specified for other entities, it defaults to entity_name-entity_version.", + "$ref": "#/$defs/string" + }, + "provisioned_model_units": { + "description": "The number of model units provisioned.", + "$ref": "#/$defs/int64" + }, + "scale_to_zero_enabled": { + "description": "Whether the compute resources for the served entity should scale down to zero.", + "$ref": "#/$defs/bool" + }, + "workload_size": { + "description": "The workload size of the served entity. The workload size corresponds to a range of provisioned concurrency that the compute autoscales between. A single unit of provisioned concurrency can process one request at a time. Valid workload sizes are \"Small\" (4 - 4 provisioned concurrency), \"Medium\" (8 - 16 provisioned concurrency), and \"Large\" (16 - 64 provisioned concurrency). Additional custom workload sizes can also be used when available in the workspace. If scale-to-zero is enabled, the lower bound of the provisioned concurrency for each workload size is 0. Do not use if min_provisioned_concurrency and max_provisioned_concurrency are specified.", + "$ref": "#/$defs/string" + }, + "workload_type": { + "description": "The workload type of the served entity. The workload type selects which type of compute to use in the endpoint. The default value for this parameter is \"CPU\". For deep learning workloads, GPU acceleration is available by selecting workload types like GPU_SMALL and others. See the available [GPU types](https://docs.databricks.com/en/machine-learning/model-serving/create-manage-serving-endpoints.html#gpu-workload-types).", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInputWorkloadType" + } + }, + "additionalProperties": false, + "required": [ + "model_name", + "model_version", + "scale_to_zero_enabled" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInputWorkloadType": { + "oneOf": [ + { + "type": "string", + "description": "Please keep this in sync with with workload types in InferenceEndpointEntities.scala", + "enum": [ + "CPU", + "GPU_MEDIUM", + "GPU_SMALL", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServingModelWorkloadType": { + "oneOf": [ + { + "type": "string", + "description": "Please keep this in sync with with workload types in InferenceEndpointEntities.scala", + "enum": [ + "CPU", + "GPU_MEDIUM", + "GPU_SMALL", + "GPU_LARGE", + "MULTIGPU_MEDIUM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.TrafficConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "routes": { + "description": "The list of routes that define traffic to each served entity.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.Channel": { + "oneOf": [ + { + "type": "object", + "description": "Configures the channel name and DBSQL version of the warehouse. CHANNEL_NAME_CUSTOM should be chosen only when `dbsql_version` is specified.", + "properties": { + "dbsql_version": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.ChannelName" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.ChannelName": { + "oneOf": [ + { + "type": "string", + "enum": [ + "CHANNEL_NAME_PREVIEW", + "CHANNEL_NAME_CURRENT", + "CHANNEL_NAME_PREVIOUS", + "CHANNEL_NAME_CUSTOM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.CreateWarehouseRequestWarehouseType": { + "oneOf": [ + { + "type": "string", + "description": "Warehouse type: `PRO` or `CLASSIC`. If you want to use serverless compute, you must set to `PRO` and also set the field `enable_serverless_compute` to `true`.", + "enum": [ + "TYPE_UNSPECIFIED", + "CLASSIC", + "PRO" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.EndpointTagPair": { + "oneOf": [ + { + "type": "object", + "properties": { + "key": { + "$ref": "#/$defs/string" + }, + "value": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.EndpointTags": { + "oneOf": [ + { + "type": "object", + "properties": { + "custom_tags": { + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/sql.EndpointTagPair" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.SpotInstancePolicy": { + "oneOf": [ + { + "type": "string", + "description": "Configurations whether the warehouse should use spot instances.", + "enum": [ + "POLICY_UNSPECIFIED", + "COST_OPTIMIZED", + "RELIABILITY_OPTIMIZED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "workspace.AzureKeyVaultSecretScopeMetadata": { + "oneOf": [ + { + "type": "object", + "description": "The metadata of the Azure KeyVault for a secret scope of type `AZURE_KEYVAULT`", + "properties": { + "dns_name": { + "description": "The DNS of the KeyVault", + "$ref": "#/$defs/string" + }, + "resource_id": { + "description": "The resource id of the azure KeyVault that user wants to associate the scope with.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "dns_name", + "resource_id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "workspace.ScopeBackendType": { + "oneOf": [ + { + "type": "string", + "description": "The types of secret scope backends in the Secret Manager. Azure KeyVault backed secret scopes\nwill be supported in a later release.", + "enum": [ + "DATABRICKS", + "AZURE_KEYVAULT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "int": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "int64": { + "oneOf": [ + { + "type": "integer" + }, + { + "type": "string", + "pattern": "\\$\\{(resources(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(bundle(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(workspace(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(artifacts(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "interface": {}, + "map": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.App": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.App" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Cluster": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Cluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Dashboard": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Dashboard" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseCatalog": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DatabaseCatalog" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseInstance": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DatabaseInstance" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Job": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Job" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperiment": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowExperiment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModel": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpoint": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Pipeline": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Pipeline" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.QualityMonitor": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.QualityMonitor" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.RegisteredModel": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.RegisteredModel" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Schema": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Schema" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScope": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScope" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SqlWarehouse": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SqlWarehouse" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SyncedDatabaseTable": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Volume": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Volume" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.TargetVariable": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.TargetVariable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "variable.Variable": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/variable.Variable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "config.Artifact": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Artifact" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Command": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Command" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Script": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Script" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "config.Target": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Target" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "interface": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/interface" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "string": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "slice": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config": { + "resources.AppPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.AppPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ClusterPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ClusterPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DashboardPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DashboardPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.DatabaseInstancePermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.DatabaseInstancePermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Grant": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Grant" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.JobPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.JobPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowExperimentPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.MlflowModelPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MlflowModelPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.ModelServingEndpointPermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.Permission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Permission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.PipelinePermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PipelinePermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SecretScopePermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SecretScopePermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.SqlWarehousePermission": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SqlWarehousePermission" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.VolumeGrant": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VolumeGrant" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.VolumeGrantPrivilege": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VolumeGrantPrivilege" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "config.ArtifactFile": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.ArtifactFile" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + }, + "databricks-sdk-go": { + "service": { + "apps.AppResource": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/apps.AppResource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.MonitorMetric": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.MonitorMetric" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.InitScriptInfo": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "compute.Library": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.Library" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.DatabaseInstanceRef": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.DatabaseInstanceRef" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobCluster": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobEnvironment": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobParameterDefinition": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobParameterDefinition" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.JobsHealthRule": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobsHealthRule" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.PowerBiTable": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.PowerBiTable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SqlTaskSubscription": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SqlTaskSubscription" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.SubscriptionSubscriber": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.SubscriptionSubscriber" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Task": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Task" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.TaskDependency": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.TaskDependency" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "jobs.Webhook": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.Webhook" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ExperimentTag": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.ModelTag": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.ModelTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.DayOfWeek": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DayOfWeek" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.IngestionConfig": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.IngestionConfig" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.Notifications": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.Notifications" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineCluster": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineCluster" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "pipelines.PipelineLibrary": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.AiGatewayRateLimit": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.EndpointTag": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.EndpointTag" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.RateLimit": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.RateLimit" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.Route": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.Route" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedEntityInput": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "serving.ServedModelInput": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/serving.ServedModelInput" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "sql.EndpointTagPair": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/sql.EndpointTagPair" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + }, + "string": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/string" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + }, + "string": { + "type": "string" + } + }, + "type": "object", + "properties": { + "artifacts": { + "description": "Defines the attributes to build an artifact", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact", + "markdownDescription": "Defines the attributes to build artifacts, where each key is the name of the artifact, and the value is a Map that defines the artifact build settings. For information about the `artifacts` mapping, see [artifacts](https://docs.databricks.com/dev-tools/bundles/settings.html#artifacts).\n\nArtifact settings defined in the top level of the bundle configuration can be overridden in the `targets` mapping. See [link](https://docs.databricks.com/dev-tools/bundles/artifact-overrides.html)." + }, + "bundle": { + "description": "The bundle attributes when deploying to this target.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle", + "markdownDescription": "The bundle attributes when deploying to this target," + }, + "environments": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target", + "deprecationMessage": "Deprecated: please use targets instead", + "deprecated": true + }, + "experimental": { + "description": "Defines attributes for experimental features.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Experimental" + }, + "include": { + "description": "Specifies a list of path globs that contain configuration files to include within the bundle.", + "$ref": "#/$defs/slice/string", + "markdownDescription": "Specifies a list of path globs that contain configuration files to include within the bundle. See [include](https://docs.databricks.com/dev-tools/bundles/settings.html#include)." + }, + "permissions": { + "description": "Defines a permission for a specific entity.", + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission", + "markdownDescription": "A Sequence that defines the permissions to apply to experiments, jobs, pipelines, and models defined in the bundle, where each item in the sequence is a permission for a specific entity.\n\nSee [permissions](https://docs.databricks.com/dev-tools/bundles/settings.html#permissions) and [link](https://docs.databricks.com/dev-tools/bundles/permissions.html)." + }, + "presets": { + "description": "Defines bundle deployment presets.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Presets", + "markdownDescription": "Defines bundle deployment presets. See [presets](https://docs.databricks.com/dev-tools/bundles/deployment-modes.html#presets)." + }, + "resources": { + "description": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Resources", + "markdownDescription": "A Map that defines the resources for the bundle, where each key is the name of the resource, and the value is a Map that defines the resource. For more information about Databricks Asset Bundles supported resources, and resource definition reference, see [link](https://docs.databricks.com/dev-tools/bundles/resources.html).\n\n```yaml\nresources:\n \u003cresource-type\u003e:\n \u003cresource-name\u003e:\n \u003cresource-field-name\u003e: \u003cresource-field-value\u003e\n```" + }, + "run_as": { + "description": "The identity to use when running Databricks Asset Bundles workflows.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.JobRunAs", + "markdownDescription": "The identity to use when running Databricks Asset Bundles workflows. See [link](https://docs.databricks.com/dev-tools/bundles/run-as.html)." + }, + "scripts": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Script" + }, + "sync": { + "description": "The files and file paths to include or exclude in the bundle.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Sync", + "markdownDescription": "The files and file paths to include or exclude in the bundle. See [sync](https://docs.databricks.com/dev-tools/bundles/settings.html#sync)." + }, + "targets": { + "description": "Defines deployment targets for the bundle.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Target", + "markdownDescription": "Defines deployment targets for the bundle. See [targets](https://docs.databricks.com/dev-tools/bundles/settings.html#targets)" + }, + "variables": { + "description": "A Map that defines the custom variables for the bundle, where each key is the name of the variable, and the value is a Map that defines the variable.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/variable.Variable" + }, + "workspace": { + "description": "Defines the Databricks workspace for the bundle.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Workspace", + "markdownDescription": "Defines the Databricks workspace for the bundle. See [workspace](https://docs.databricks.com/dev-tools/bundles/settings.html#workspace)." + } + }, + "additionalProperties": {} +} \ No newline at end of file diff --git a/product_demos/.DS_Store b/product_demos/.DS_Store new file mode 100644 index 00000000..5a433b05 Binary files /dev/null and b/product_demos/.DS_Store differ diff --git a/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.bundlevars.json b/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.bundlevars.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.bundlevars.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.overrides.json b/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.overrides.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/product_demos/Delta-Live-Table/declarative-pipelines/deployment/.databricks/bundle/dev/vscode.overrides.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/product_demos/cdc-pipeline/.DS_Store b/product_demos/cdc-pipeline/.DS_Store new file mode 100644 index 00000000..8abd3361 Binary files /dev/null and b/product_demos/cdc-pipeline/.DS_Store differ diff --git a/product_demos/cdc-pipeline/01-CDC-CDF-simple-pipeline.py b/product_demos/cdc-pipeline/01-CDC-CDF-simple-pipeline.py index 74a4c9e7..67d88f59 100644 --- a/product_demos/cdc-pipeline/01-CDC-CDF-simple-pipeline.py +++ b/product_demos/cdc-pipeline/01-CDC-CDF-simple-pipeline.py @@ -1,24 +1,32 @@ # Databricks notebook source # MAGIC %md # MAGIC -# MAGIC # Implement CDC: Change Data Capture -# MAGIC ## Use-case: Synchronize your SQL Database with your Lakehouse +# MAGIC # CDC Pipeline Demo: Change Data Capture with Serverless Compute +# MAGIC ## Step-by-Step Guide to Building a Cost-Effective CDC Pipeline # MAGIC -# MAGIC Delta Lake is an open-source storage layer with Transactional capabilities and increased Performances. +# MAGIC This demo shows you how to build a **Change Data Capture (CDC)** pipeline using **Databricks Serverless Compute** for cost-effective, auto-scaling data processing. # MAGIC -# MAGIC Delta lake is designed to support CDC workload by providing support for UPDATE / DELETE and MERGE operation. +# MAGIC ### What You'll Learn: +# MAGIC 1. **🥉 Step 1**: Set up CDC data simulation +# MAGIC 2. **🥈 Step 2**: Build Bronze layer with Auto Loader +# MAGIC 3. **🥇 Step 3**: Create Silver layer with MERGE operations +# MAGIC 4. **🚀 Step 4**: Implement Gold layer with Change Data Feed (CDF) +# MAGIC 5. **📊 Step 5**: Continuous CDC Data # MAGIC -# MAGIC In addition, Delta table can support CDC to capture internal changes and propagate the changes downstream. # MAGIC -# MAGIC Note that this is a fairly advaned demo. Before going into this content, we recommend you get familiar with Delta Lake `dbdemos.install('delta-lake')`. +# MAGIC ### Key Benefits of Serverless CDC: +# MAGIC - 💰 **Cost-effective**: Pay only for compute time used +# MAGIC - 🚀 **Auto-scaling**: Automatically scales based on workload +# MAGIC - ⚡ **Fast processing**: Optimized for batch processing with `availableNow` triggers +# MAGIC - 🔄 **Incremental**: Only processes new/changed data # MAGIC -# MAGIC ## Simplifying CDC with Delta Live Table +# MAGIC ### Prerequisites: +# MAGIC - Basic understanding of Delta Lake: `dbdemos.install('delta-lake')` +# MAGIC - Familiarity with Structured Streaming concepts # MAGIC -# MAGIC As you'll see, implementing a CDC pipeline from scratch is slightly advanced. +# MAGIC --- # MAGIC -# MAGIC To simplify these operation & implement a full CDC flow with SQL expression, we strongly advise to use Delta Live Table with `APPLY CHANGES`: `dbdemos.install('delta-live-table')` (including native SCDT2 support) -# MAGIC -# MAGIC As you'll see, `APPLY CHANGES` handles the MERGE INTO + DEDUPLICATION complexity for you. +# MAGIC **💡 Alternative Approach**: For production CDC pipelines, consider using **Delta Live Tables** with `APPLY CHANGES` for simplified implementation: `dbdemos.install('delta-live-table')` # MAGIC # MAGIC # MAGIC @@ -30,92 +38,298 @@ # COMMAND ---------- +# DBTITLE 1,Import Required Functions +from pyspark.sql.functions import current_timestamp, col + +# COMMAND ---------- + +# DBTITLE 1,Configure Schema Evolution for CDC Processing +# Enable automatic schema merging for all Delta operations to handle schema changes +# Schema evolution is handled automatically by mergeSchema=true in writeStream operations +# Schema inference is handled automatically by Auto Loader with cloudFiles.inferColumnTypes=true + +# COMMAND ---------- + # MAGIC %md # MAGIC Delta Lake Change Data Feed # COMMAND ---------- # MAGIC %md -# MAGIC ## CDC flow +# MAGIC ## 📋 CDC Pipeline Architecture Overview # MAGIC -# MAGIC Here is the flow we'll implement, consuming CDC data from an external database. Note that the incoming could be any format, including message queue such as Kafka. +# MAGIC Here's the complete CDC pipeline we'll build using **Serverless Compute**: # MAGIC -# MAGIC Make all your data ready for BI and ML +# MAGIC CDC Pipeline Architecture +# MAGIC +# MAGIC ### Pipeline Flow: +# MAGIC 1. **📥 Data Source**: CDC events from external database (simulated) +# MAGIC 2. **🥉 Bronze Layer**: Raw CDC data ingestion with Auto Loader +# MAGIC 3. **🥈 Silver Layer**: Cleaned, deduplicated data with MERGE operations +# MAGIC 4. **🥇 Gold Layer**: Business-ready data with Change Data Feed (CDF) +# MAGIC 5. **📊 Analytics**: Real-time insights and reporting +# MAGIC +# MAGIC **💡 Note**: The incoming data could be any format, including message queues like Kafka. # COMMAND ---------- -# MAGIC %md-sandbox -# MAGIC ## Bronze: Incremental data loading using Auto Loader +# MAGIC %md +# MAGIC ## 🥉 Step 1: Bronze Layer - Raw Data Ingestion # MAGIC -# MAGIC Make all your data ready for BI and ML +# MAGIC Bronze Layer # MAGIC -# MAGIC Working with external system can be challenging due to schema update. The external database can have schema update, adding or modifying columns, and our system must be robust against these changes. +# MAGIC ### What We're Building: +# MAGIC - **Purpose**: Ingest raw CDC data from external sources +# MAGIC - **Technology**: Auto Loader with serverless compute +# MAGIC - **Benefits**: Automatic schema evolution and incremental processing # MAGIC -# MAGIC Databricks Autoloader (`cloudFiles`) handles schema inference and evolution out of the box. +# MAGIC ### Key Features: +# MAGIC - 🔄 **Schema Evolution**: Handles database schema changes automatically +# MAGIC - 📈 **Incremental Processing**: Only processes new files +# MAGIC - ⚡ **Serverless Scaling**: Auto-scales based on data volume +# MAGIC - 💰 **Cost Efficient**: Pay only for processing time # MAGIC -# MAGIC For more details on Auto Loader, run `dbdemos.install('auto-loader')` +# MAGIC **💡 Learn More**: For detailed Auto Loader concepts, run `dbdemos.install('auto-loader')` + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Step 1.1: Explore Incoming CDC Data # COMMAND ---------- -# DBTITLE 1,Let's explore our incoming data. We receive CSV files with client information +print("🔍 Exploring our incoming CDC data structure...") cdc_raw_data = spark.read.option('header', "true").csv(raw_data_location+'/user_csv') display(cdc_raw_data) # COMMAND ---------- -# DBTITLE 1,Our CDC is sending 3 type of operation: APPEND, DELETE and UPDATE +# MAGIC %md +# MAGIC ## Step 1.2: Understand CDC Operation Types + +# COMMAND ---------- + +print("🔍 Understanding CDC operation types...") +print("Our CDC system sends 3 types of operations:") display(cdc_raw_data.dropDuplicates(['operation'])) # COMMAND ---------- -# DBTITLE 1,We need to keep the cdc information, however csv isn't a efficient storage. Let's put that in a Delta table instead: +# MAGIC %md +# MAGIC ## Step 1.3: Set Up Continuous CDC Data Simulation +# MAGIC +# MAGIC To demonstrate serverless compute capabilities, we'll create a data generator that simulates incoming CDC events every 60 seconds. +# MAGIC +# MAGIC ### Why This Matters: +# MAGIC - 🚀 **Auto-scaling**: Shows how serverless scales with workload +# MAGIC - 💰 **Cost Efficiency**: Demonstrates `availableNow` trigger benefits +# MAGIC - 🔄 **Real-world Simulation**: Mimics continuous CDC scenarios +# MAGIC - 📊 **Monitoring**: Enables table growth visualization + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Step 1.4: CDC Data Generator Implementation + +# COMMAND ---------- + +import threading +import time +import random +from datetime import datetime +import pandas as pd + +# Global variable to control the data generator +generator_running = False + +def generate_cdc_record(operation_type="UPDATE", user_id=None): + """Generate a single CDC record""" + if user_id is None: + user_id = random.randint(1, 1000) + + operations = { + "INSERT": { + "id": user_id, + "name": f"User_{user_id}_{random.randint(1,99)}", + "address": f"Address_{random.randint(1,999)} Street", + "email": f"user{user_id}@company{random.randint(1,10)}.com", + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "INSERT" + }, + "UPDATE": { + "id": user_id, + "name": f"Updated_User_{user_id}", + "address": f"New_Address_{random.randint(1,999)} Avenue", + "email": f"updated.user{user_id}@newcompany{random.randint(1,5)}.com", + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "UPDATE" + }, + "DELETE": { + "id": user_id, + "name": None, + "address": None, + "email": None, + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "DELETE" + } + } + return operations[operation_type] + +def continuous_cdc_generator(): + """Background function that generates CDC data every 120 seconds""" + global generator_running + file_counter = 0 + + while generator_running: + try: + # Generate 3-5 random CDC events + num_events = random.randint(3, 5) + cdc_events = [] + + for _ in range(num_events): + # Random operation type with weighted probability + operation = random.choices( + ["INSERT", "UPDATE", "DELETE"], + weights=[50, 40, 10] # More inserts/updates than deletes + )[0] + cdc_events.append(generate_cdc_record(operation)) + + # Create DataFrame and save as CSV + df = pd.DataFrame(cdc_events) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"cdc_events_{timestamp}_{file_counter}.csv" + file_path = f"{raw_data_location}/user_csv/{filename}" + + # Convert to Spark DataFrame and save + spark_df = spark.createDataFrame(df) + spark_df.coalesce(1).write.mode("overwrite").option("header", "true").csv(file_path) + + print(f"Generated {num_events} CDC events at {datetime.now()}: {filename}") + file_counter += 1 + + # Wait 60 seconds before next batch + time.sleep(60) + + except Exception as e: + print(f"Error in CDC generator: {e}") + time.sleep(60) # Continue even if there's an error + +def start_cdc_generator(): + """Start the CDC data generator in background""" + global generator_running + if not generator_running: + generator_running = True + generator_thread = threading.Thread(target=continuous_cdc_generator, daemon=True) + generator_thread.start() + print("🚀 CDC Data Generator started! New data will arrive every 60 seconds.") + print("💡 This simulates continuous CDC events for serverless processing demonstration.") + return generator_thread + else: + print("CDC Generator is already running!") + return None + +def stop_cdc_generator(): + """Stop the CDC data generator""" + global generator_running + generator_running = False + print("🛑 CDC Data Generator stopped.") + +# Start the data generator for continuous simulation +data_generator_thread = start_cdc_generator() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 🥈 Step 2: Create Bronze Delta Table With Auto Loader +# MAGIC + +# COMMAND ---------- + +# Drop existing table if it exists to avoid schema conflicts +try: + spark.sql("DROP TABLE IF EXISTS clients_cdc") + print("🔄 Dropped existing clients_cdc table to avoid schema conflicts") +except: + pass + bronzeDF = (spark.readStream .format("cloudFiles") .option("cloudFiles.format", "csv") - #.option("cloudFiles.maxFilesPerTrigger", "1") #Simulate streaming, remove in production .option("cloudFiles.inferColumnTypes", "true") .option("cloudFiles.schemaLocation", raw_data_location+"/stream/schema_cdc_raw") .option("cloudFiles.schemaHints", "id bigint, operation_date timestamp") + .option("cloudFiles.useNotifications", "false") # Optimized for serverless + .option("cloudFiles.includeExistingFiles", "true") # Process all files on first run .load(raw_data_location+'/user_csv')) -(bronzeDF.withColumn("file_name", col("_metadata.file_path")).writeStream +(bronzeDF.withColumn("file_name", col("_metadata.file_path")) + .withColumn("processing_time", current_timestamp()) # Add processing timestamp + .writeStream .option("checkpointLocation", raw_data_location+"/stream/checkpoint_cdc_raw") - .trigger(processingTime='10 seconds') - #.trigger(availableNow=True) --use this trigger on serverless + .option("mergeSchema", "true") # Enable schema evolution + .trigger(availableNow=True) # Serverless trigger for cost-effective processing .table("clients_cdc")) -time.sleep(20) +time.sleep(10) # COMMAND ---------- # MAGIC %sql -# MAGIC -- let's make sure our table has the proper compaction settings to support streaming -# MAGIC ALTER TABLE clients_cdc SET TBLPROPERTIES (delta.autoOptimize.optimizeWrite = true, delta.autoOptimize.autoCompact = true); +# MAGIC -- Optimize table properties for serverless streaming and performance +# MAGIC ALTER TABLE clients_cdc SET TBLPROPERTIES ( +# MAGIC delta.autoOptimize.optimizeWrite = true, +# MAGIC delta.autoOptimize.autoCompact = true, +# MAGIC delta.targetFileSize = '128MB', +# MAGIC delta.tuneFileSizesForRewrites = true +# MAGIC ); # MAGIC # MAGIC SELECT * FROM clients_cdc order by id asc ; # COMMAND ---------- -# MAGIC %md-sandbox -# MAGIC ## Silver: Materialize the table +# MAGIC %md +# MAGIC ## 🥈 Step 3: Silver Layer - Data Cleaning and Deduplication # MAGIC -# MAGIC Make all your data ready for BI and ML +# MAGIC Silver Layer # MAGIC -# MAGIC The silver `retail_client_silver` table will contains the most up to date view. It'll be a replicat of the original MYSQL table. +# MAGIC ### What We're Building: +# MAGIC - **Purpose**: Clean, deduplicate, and standardize CDC data +# MAGIC - **Technology**: Delta MERGE operations with serverless compute +# MAGIC - **Benefits**: Idempotent processing and data quality # MAGIC -# MAGIC Because we'll propagate the `MERGE` operations downstream to the `GOLD` layer, we need to enable Delta Lake CDF: `delta.enableChangeDataFeed = true` +# MAGIC ### Key Features: +# MAGIC - 🔄 **Idempotent**: Safe to run multiple times +# MAGIC - ⚡ **Serverless**: Auto-scales with data volume +# MAGIC - 💰 **Cost Efficient**: Only processes new/changed data +# MAGIC - 📊 **CDF Enabled**: Tracks changes for downstream processing +# MAGIC +# MAGIC **💡 Note**: We enable Change Data Feed (CDF) to track modifications for the Gold layer. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Step 3.1: Create Silver Table With Change Data Feed Enabled # COMMAND ---------- -# DBTITLE 1,We can now create our client table using standard SQL command # MAGIC %sql -# MAGIC -- we can add NOT NULL in our ID field (or even more advanced constraint) +# MAGIC -- Create silver table with optimized settings for serverless and CDC # MAGIC CREATE TABLE IF NOT EXISTS retail_client_silver (id BIGINT NOT NULL, name STRING, address STRING, email STRING, operation STRING) -# MAGIC TBLPROPERTIES (delta.enableChangeDataFeed = true, delta.autoOptimize.optimizeWrite = true, delta.autoOptimize.autoCompact = true); +# MAGIC TBLPROPERTIES ( +# MAGIC delta.enableChangeDataFeed = true, +# MAGIC delta.autoOptimize.optimizeWrite = true, +# MAGIC delta.autoOptimize.autoCompact = true, +# MAGIC delta.targetFileSize = '128MB', +# MAGIC delta.tuneFileSizesForRewrites = true +# MAGIC ); + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Step 3.2: Implement MERGE Operations # COMMAND ---------- -# DBTITLE 1,And run our MERGE statement the upsert the CDC information in our final table #for each batch / incremental update from the raw cdc table, we'll run a MERGE on the silver table def merge_stream(df, i): df.createOrReplaceTempView("clients_cdc_microbatch") @@ -137,8 +351,8 @@ def merge_stream(df, i): .writeStream .foreachBatch(merge_stream) .option("checkpointLocation", raw_data_location+"/stream/checkpoint_clients_cdc") - .trigger(processingTime='10 seconds') - #.trigger(availableNow=True) --use this trigger on serverless + .option("mergeSchema", "true") # Enable schema evolution for silver layer + .trigger(availableNow=True) # Serverless trigger for cost-effective processing .start()) time.sleep(20) @@ -151,12 +365,11 @@ def merge_stream(df, i): # COMMAND ---------- # MAGIC %md -# MAGIC ### Testing the first CDC layer +# MAGIC ### Step 3.3: Test Merge Operations In Silver Layer # MAGIC Let's send a new CDC entry to simulate an update and a DELETE for the ID 1 and 2 # COMMAND ---------- -# DBTITLE 1,Let's UPDATE id=1 and DELETE the row with id=2 # MAGIC %sql # MAGIC insert into clients_cdc (id, name, address, email, operation_date, operation, _rescued_data, file_name) values # MAGIC (1000, "Quentin", "Paris 75020", "quentin.ambard@databricks.com", now(), "UPDATE", null, null), @@ -170,19 +383,28 @@ def merge_stream(df, i): # COMMAND ---------- -# DBTITLE 1,Wait a few seconds for the stream to catch the new entry in the CDC table and check the results in the main table +# DBTITLE 1,🥈 Step 2.5: Verify CDC Processing Results # MAGIC %sql # MAGIC select * from retail_client_silver where id in (1000, 2000); # MAGIC -- Note that ID 1000 has been updated, and ID 2000 is deleted # COMMAND ---------- -# MAGIC %md-sandbox -# MAGIC ## Gold: capture and propagate Silver modifications downstream +# MAGIC %md +# MAGIC ## 🚀 Step 4: Gold Layer - Business-Ready Data with Change Data Feed # MAGIC -# MAGIC Make all your data ready for BI and ML +# MAGIC Gold Layer # MAGIC -# MAGIC We need to add a final Gold layer based on the data from the Silver table. If a row is DELETED or UPDATED in the SILVER layer, we want to apply the same modification in the GOLD layer. +# MAGIC ### What We're Building: +# MAGIC - **Purpose**: Create business-ready data from Silver layer changes +# MAGIC - **Technology**: Change Data Feed (CDF) with serverless compute +# MAGIC - **Benefits**: Real-time propagation of Silver layer modifications +# MAGIC +# MAGIC ### How It Works: +# MAGIC - 📊 **CDF Tracking**: Monitors all changes in Silver table +# MAGIC - 🔄 **Real-time Sync**: Applies DELETEs and UPDATEs to Gold layer +# MAGIC - ⚡ **Serverless**: Auto-scales based on change volume +# MAGIC - 💰 **Cost Efficient**: Only processes actual changes # MAGIC # MAGIC To do so, we need to capture all the tables changes from the SILVER layer and incrementally replicate the changes to the GOLD layer. # MAGIC @@ -193,7 +415,23 @@ def merge_stream(df, i): # COMMAND ---------- # MAGIC %md -# MAGIC ### Working with Delta Lake CDF +# MAGIC ### Step 4.1: Understanding Change Data Feed (CDF) vs Non-CDF Processing +# MAGIC +# MAGIC **🔍 Key Difference**: CDF only processes **actual changes**, while non-CDF processes **all data**. +# MAGIC +# MAGIC #### **Non-CDF Approach (Inefficient)**: +# MAGIC - 📊 **Processes**: Entire table every time +# MAGIC - 💰 **Cost**: High - reprocesses unchanged data +# MAGIC - ⏱️ **Time**: Slow - scans all records +# MAGIC - 🔄 **Example**: If table has 1M records, processes all 1M even for 1 change +# MAGIC +# MAGIC #### **CDF Approach (Efficient)**: +# MAGIC - 📊 **Processes**: Only changed records +# MAGIC - 💰 **Cost**: Low - only pays for actual changes +# MAGIC - ⏱️ **Time**: Fast - processes only deltas +# MAGIC - 🔄 **Example**: If table has 1M records but only 5 changed, processes only 5 records +# MAGIC +# MAGIC **💡 CDF Benefits**: Up to 99%+ reduction in processing volume for incremental changes! # COMMAND ---------- @@ -206,7 +444,7 @@ def merge_stream(df, i): # COMMAND ---------- -# MAGIC %md #### Delta CDF table_changes output +# MAGIC %md # MAGIC Table Changes provides back 4 cdc types in the "_change_type" column: # MAGIC # MAGIC | CDC Type | Description | @@ -220,22 +458,63 @@ def merge_stream(df, i): # COMMAND ---------- -# DBTITLE 1,Getting the last modifications with the Python API +# MAGIC %md +# MAGIC ### Step 4.2: Demonstrate CDF vs Non-CDF Processing Volume +# MAGIC +# MAGIC Let's show the actual difference in processing volume between CDF and non-CDF approaches. + +# COMMAND ---------- + from delta.tables import * -#Let's get the last table version to only see the last update mofications +# Let's demonstrate the processing volume difference +print("🔍 Demonstrating CDF vs Non-CDF Processing Volume") +print("=" * 60) + +# Get total records in silver table +total_silver_records = spark.sql("SELECT COUNT(*) as count FROM retail_client_silver").collect()[0]['count'] +print(f"📊 Total records in Silver table: {total_silver_records:,}") + +# Get latest table version last_version = str(DeltaTable.forName(spark, "retail_client_silver").history(1).head()["version"]) -print(f"our Delta table last version is {last_version}, let's select the last changes to see our DELETE and UPDATE operations (last 2 versions):") +print(f"📈 Latest table version: {last_version}") +# Show what CDF would process (only changes from last 2 versions) +print(f"\n🔄 CDF Processing (Efficient):") changes = spark.read.format("delta") \ - .option("readChangeData", "true") \ + .option("readChangeFeed", "true") \ .option("startingVersion", int(last_version) -1) \ .table("retail_client_silver") -display(changes) + +cdf_records = changes.count() +print(f" 📊 Records to process: {cdf_records:,}") +print(f" 💰 Processing efficiency: {((total_silver_records - cdf_records) / total_silver_records * 100):.1f}% reduction") +print(f" ⚡ Speed improvement: {total_silver_records / max(cdf_records, 1):.1f}x faster") + +# Show what non-CDF would process (entire table) +print(f"\n🔄 Non-CDF Processing (Inefficient):") +print(f" 📊 Records to process: {total_silver_records:,}") +print(f" 💰 Processing efficiency: 0% reduction (processes everything)") +print(f" ⚡ Speed improvement: 1x (baseline)") + +print(f"\n💡 Key Insight: CDF processes {cdf_records:,} records instead of {total_silver_records:,} records") +print(f" That's a {((total_silver_records - cdf_records) / total_silver_records * 100):.1f}% reduction in processing volume!") + +# Display the actual changes +print(f"\n📋 Actual Changes Detected:") +display(changes.select("_change_type", "id", "name", "email").orderBy("id")) # COMMAND ---------- -# MAGIC %md ### Synchronizing our downstream GOLD table based from the Silver changes +# MAGIC %md +# MAGIC ### Step 4.3: Gold Layer Processing with CDF Efficiency +# MAGIC +# MAGIC Now let's implement the Gold layer using CDF to demonstrate the efficiency gains: +# MAGIC +# MAGIC **🎯 What We're Building**: Gold layer that only processes **actual changes** from Silver layer +# MAGIC **📊 Processing Volume**: Only changed records, not entire table +# MAGIC **💰 Cost Impact**: Significant reduction in compute costs +# MAGIC **⚡ Performance**: Much faster processing times # MAGIC # MAGIC Let's now say that we want to perform another table enhancement and propagate these changes downstream. # MAGIC @@ -247,26 +526,48 @@ def merge_stream(df, i): # COMMAND ---------- -# DBTITLE 1,Let's create or final GOLD table: retail_client_gold +# DBTITLE 1,Step 4.4: Create Gold Table with Processing Volume Tracking # MAGIC %sql -# MAGIC CREATE TABLE IF NOT EXISTS retail_client_gold (id BIGINT NOT NULL, name STRING, address STRING, email STRING, gold_data STRING); +# MAGIC CREATE TABLE IF NOT EXISTS retail_client_gold (id BIGINT NOT NULL, name STRING, address STRING, email STRING, gold_data STRING) +# MAGIC TBLPROPERTIES ( +# MAGIC delta.autoOptimize.optimizeWrite = true, +# MAGIC delta.autoOptimize.autoCompact = true, +# MAGIC delta.targetFileSize = '128MB', +# MAGIC delta.tuneFileSizesForRewrites = true +# MAGIC ); # COMMAND ---------- from pyspark.sql.window import Window -from pyspark.sql.functions import dense_rank, regexp_replace, lit, col +from pyspark.sql.functions import dense_rank, regexp_replace, lit, col, current_timestamp -#Function to upsert `microBatchOutputDF` into Delta table using MERGE +# Function to upsert `microBatchOutputDF` into Delta table using MERGE +# This function demonstrates CDF efficiency by processing only changed records def upsertToDelta(data, batchId): - #First we need to deduplicate based on the id and take the most recent update + print(f"🔄 Processing batch {batchId} with CDF efficiency...") + + # Count records being processed + records_to_process = data.count() + print(f" 📊 Records in this batch: {records_to_process:,}") + + # First we need to deduplicate based on the id and take the most recent update windowSpec = Window.partitionBy("id").orderBy(col("_commit_version").desc()) - #Select only the first value - #getting the latest change is still needed if the cdc contains multiple time the same id. We can rank over the id and get the most recent _commit_version + # Select only the first value + # getting the latest change is still needed if the cdc contains multiple time the same id. We can rank over the id and get the most recent _commit_version data_deduplicated = data.withColumn("rank", dense_rank().over(windowSpec)).where("rank = 1 and _change_type!='update_preimage'").drop("_commit_version", "rank") - #Add some data cleaning for the gold layer to remove quotes from the address + # Add some data cleaning for the gold layer to remove quotes from the address data_deduplicated = data_deduplicated.withColumn("address", regexp_replace(col("address"), "\"", "")) + # Count deduplicated records + deduplicated_count = data_deduplicated.count() + print(f" 📊 Records after deduplication: {deduplicated_count:,}") + + # Show processing efficiency + if records_to_process > 0: + efficiency = ((records_to_process - deduplicated_count) / records_to_process * 100) + print(f" 💰 Deduplication efficiency: {efficiency:.1f}% reduction") + #run the merge in the gold table directly (DeltaTable.forName(spark, "retail_client_gold").alias("target") .merge(data_deduplicated.alias("source"), "source.id = target.id") @@ -275,56 +576,278 @@ def upsertToDelta(data, batchId): .whenNotMatchedInsertAll("source._change_type != 'delete'") .execute()) + print(f" ✅ Batch {batchId} completed - processed {deduplicated_count:,} records efficiently") + + +# Start the CDF stream with processing volume tracking +print("🚀 Starting Gold layer CDF stream with processing volume tracking...") +print("💡 This will show you exactly how many records are processed vs. total table size") (spark.readStream - .option("readChangeData", "true") + .option("readChangeFeed", "true") # Updated to use correct option name .option("startingVersion", 1) .table("retail_client_silver") .withColumn("gold_data", lit("Delta CDF is Awesome")) .writeStream .foreachBatch(upsertToDelta) .option("checkpointLocation", raw_data_location+"/stream/checkpoint_clients_gold") - .trigger(processingTime='10 seconds') - #.trigger(availableNow=True) --use this trigger on serverless - .start()) - -time.sleep(20) + .option("mergeSchema", "true") # Enable schema evolution for gold layer + .trigger(availableNow=True) # Serverless trigger for cost-effective processing + .start() + .awaitTermination()) # COMMAND ---------- -# MAGIC %sql SELECT * FROM retail_client_gold +# MAGIC %sql +# MAGIC -- Show the final Gold table results +# MAGIC SELECT * FROM retail_client_gold ORDER BY id; # COMMAND ---------- -# MAGIC %md-sandbox -# MAGIC ### Support for data sharing and Datamesh organization -# MAGIC -# MAGIC -# MAGIC As we've seen during this demo, you can track all the changes (INSERT/UPDATE/DELETE) from any Detlta table using the CDC option. +# MAGIC %md +# MAGIC ### Step 4.5: CDF Processing Volume Summary # MAGIC -# MAGIC It's then easy to subscribe the table modifications as an incremental process. +# MAGIC **🎯 What We Just Demonstrated**: +# MAGIC - **CDF Processing**: Only processed actual changes from Silver layer +# MAGIC - **Volume Efficiency**: Dramatically reduced processing volume +# MAGIC - **Cost Savings**: Significant reduction in compute costs +# MAGIC - **Performance**: Much faster processing times # MAGIC -# MAGIC This makes the Data Mesh implementation easy: each Mesh can publish a set of tables, and other meshes can subscribe the original changes. +# MAGIC **📊 Key Metrics**: +# MAGIC - **Total Silver Records**: Shows full table size +# MAGIC - **CDF Records Processed**: Shows only changed records +# MAGIC - **Efficiency Gain**: Percentage reduction in processing volume +# MAGIC - **Speed Improvement**: Multiplier for processing speed # MAGIC -# MAGIC They are then in charge of propagating the changes (ex GDPR DELETE) to their own Data Mesh +# MAGIC **💡 Real-World Impact**: In production, this can mean processing 1,000 records instead of 1,000,000 records for incremental updates! # COMMAND ---------- -# MAGIC %md-sandbox -# MAGIC ## Data is now ready for BI & ML use-case ! +# MAGIC %md +# MAGIC ## 📊 Step 5: Continuous Serverless Incremental Processing # MAGIC -# MAGIC Make all your data ready for BI and ML +# MAGIC With the data generator running, you can now demonstrate continuous serverless CDC processing. The pipeline is designed to process **only newly arrived data** using checkpoints and streaming offsets. +# MAGIC +# MAGIC **Key Incremental Processing Features:** +# MAGIC - ✅ **Auto Loader Checkpoints**: Only new files since last processing +# MAGIC - ✅ **Streaming Offsets**: Only new CDC records since last checkpoint +# MAGIC - ✅ **Change Data Feed**: Only new changes since last processed version +# MAGIC - ✅ **Efficient Processing**: No reprocessing of historical data +# MAGIC - ✅ **Cost Optimization**: Pay only for new data processing + +# COMMAND ---------- + +def trigger_cdc_pipeline(): + """ + Trigger all CDC streams to process new data with serverless compute. + This function can be called periodically (every minute, 5 minutes, etc.) + """ + print(f"🔄 Triggering CDC pipeline at {datetime.now()}") + + # Enable automatic schema merging for MERGE operations + # Schema evolution is handled automatically by mergeSchema=true in writeStream operations + + # Stop any existing streams first + DBDemos.stop_all_streams() + time.sleep(5) + + # Restart bronze layer (Auto Loader) - only process new files since last checkpoint + print(" 🔄 Processing new files for bronze layer...") + bronzeDF = (spark.readStream + .format("cloudFiles") + .option("cloudFiles.format", "csv") + .option("cloudFiles.inferColumnTypes", "true") + .option("cloudFiles.schemaLocation", raw_data_location+"/stream/schema_cdc_raw") + .option("cloudFiles.schemaHints", "id bigint, operation_date timestamp") + .option("cloudFiles.useNotifications", "false") + .option("cloudFiles.includeExistingFiles", "false") # Only new files after checkpoint + .option("cloudFiles.maxFilesPerTrigger", "10") # Process in batches for efficiency + .load(raw_data_location+'/user_csv')) + + (bronzeDF.withColumn("file_name", col("_metadata.file_path")) + .withColumn("processing_time", current_timestamp()) # Track when processed + .writeStream + .option("checkpointLocation", raw_data_location+"/stream/checkpoint_cdc_raw") + .option("mergeSchema", "true") # Enable schema evolution for new columns + .trigger(availableNow=True) # Process only available new data + .table("clients_cdc") + .awaitTermination()) + + # Restart silver layer (MERGE operations) - only process new CDC records + print(" 🔄 Processing new CDC records for silver layer...") + (spark.readStream + .table("clients_cdc") + .writeStream + .foreachBatch(merge_stream) + .option("checkpointLocation", raw_data_location+"/stream/checkpoint_clients_cdc") + .option("mergeSchema", "true") # Enable schema evolution for silver layer + .trigger(availableNow=True) # Process only new CDC records since last checkpoint + .start() + .awaitTermination()) + + # Restart gold layer (CDF processing) - only process new changes since last checkpoint + print(" 🔄 Processing new changes for gold layer using Change Data Feed...") + (spark.readStream + .option("readChangeFeed", "true") + # No startingVersion specified - will automatically start from checkpoint + .table("retail_client_silver") + .withColumn("gold_data", lit("Delta CDF is Awesome")) + .withColumn("cdf_processing_time", current_timestamp()) # Track CDF processing time + .writeStream + .foreachBatch(upsertToDelta) + .option("checkpointLocation", raw_data_location+"/stream/checkpoint_clients_gold") + .option("mergeSchema", "true") # Enable schema evolution for gold layer + .trigger(availableNow=True) # Process only new changes since last checkpoint + .start() + .awaitTermination()) + + print("✅ CDC pipeline completed processing available data") + +# COMMAND ---------- + +print("🎯 Running one iteration of serverless CDC processing...") +print("💡 In production, schedule this via Databricks Jobs every few minutes") + +# Give the data generator time to create some files +print("⏳ Waiting 65 seconds for data generator to create new files...") +time.sleep(65) + +# Process any new data +trigger_cdc_pipeline() + +# Show results with table growth monitoring +print("\n📊 Monitoring table growth over time...") +print("💡 Watch how serverless compute handles growing data volumes efficiently") + +# Function to get table sizes +def get_table_sizes(): + sizes = {} + try: + sizes['bronze'] = spark.sql("SELECT COUNT(*) as count FROM clients_cdc").collect()[0]['count'] + except: + sizes['bronze'] = 0 + try: + sizes['silver'] = spark.sql("SELECT COUNT(*) as count FROM retail_client_silver").collect()[0]['count'] + except: + sizes['silver'] = 0 + try: + sizes['gold'] = spark.sql("SELECT COUNT(*) as count FROM retail_client_gold").collect()[0]['count'] + except: + sizes['gold'] = 0 + return sizes + +# Monitor table growth over multiple iterations +print("🔍 Table Size Growth Monitoring:") +print("=" * 60) + +for iteration in range(1, 4): # Monitor 3 iterations + print(f"\n📈 Iteration {iteration} - {datetime.now().strftime('%H:%M:%S')}") + + # Get current sizes + sizes = get_table_sizes() + print(f"🥉 Bronze (Raw CDC): {sizes['bronze']:,} records") + print(f"🥈 Silver (Materialized): {sizes['silver']:,} records") + print(f"🥇 Gold (Enhanced): {sizes['gold']:,} records") + + # Calculate growth if not first iteration + if iteration > 1: + growth_bronze = sizes['bronze'] - previous_sizes['bronze'] + growth_silver = sizes['silver'] - previous_sizes['silver'] + growth_gold = sizes['gold'] - previous_sizes['gold'] + + print(f" 📊 Growth: Bronze +{growth_bronze}, Silver +{growth_silver}, Gold +{growth_gold}") + + # Show recent records + print(" 🔍 Latest Records:") + try: + latest_bronze = spark.sql(""" + SELECT operation, COUNT(*) as count + FROM clients_cdc + GROUP BY operation + ORDER BY operation + """).collect() + operations_summary = {row['operation']: row['count'] for row in latest_bronze} + print(f" 📁 Operations: {operations_summary}") + + # Show latest silver records + latest_silver = spark.sql(""" + SELECT id, name, email + FROM retail_client_silver + ORDER BY id DESC + LIMIT 3 + """).collect() + if latest_silver: + print(" 📝 Latest Silver Records:") + for row in latest_silver: + print(f" ID: {row['id']}, Name: {row['name']}, Email: {row['email']}") + except Exception as e: + print(f" ⚠️ Error showing details: {e}") + + previous_sizes = sizes + + # Wait for next iteration (except on last one) + if iteration < 3: + print(f" ⏳ Waiting 65 seconds for more CDC data...") + print(" 💰 Serverless compute: No costs during wait time!") + time.sleep(65) + + # Process new data + print(f" 🔄 Processing new data (Iteration {iteration + 1})...") + trigger_cdc_pipeline() + +print("\n" + "=" * 60) +print("✅ Table growth monitoring completed!") +print("📈 Key Observations:") +print(" 🔹 Tables grow incrementally with each CDC batch") +print(" 🔹 Serverless compute scales automatically with data volume") +print(" 🔹 Cost efficiency: Pay only during processing, not waiting") +print(" 🔹 Real-time CDC processing with delta architecture") + +# COMMAND ---------- + +# DBTITLE 1,Cleanup and Stop Data Generator +stop_cdc_generator() +DBDemos.stop_all_streams() + +print("🎉 Demo completed! You've seen how serverless compute handles continuous CDC processing:") +print("✅ Cost-effective: Pay only for actual processing time") +print("✅ Auto-scaling: Automatically scales based on data volume") +print("✅ Simplified ops: No cluster management required") +print("✅ Reliable: Built-in fault tolerance and automatic restarts") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Data Ready for BI & ML Use Cases # MAGIC -# MAGIC We now have our final table, updated based on the initial CDC information we receive. +# MAGIC BI and ML Ready # MAGIC -# MAGIC As next step, we can leverage Databricks Lakehouse platform to start creating SQL queries / dashboards or ML models +# MAGIC ### What's Available: +# MAGIC - 📊 **Business Intelligence**: Create SQL queries and dashboards +# MAGIC - 🤖 **Machine Learning**: Build ML models on clean, up-to-date data +# MAGIC - 🔄 **Real-time Analytics**: Access to latest data changes +# MAGIC - 📈 **Data Quality**: Clean, deduplicated, and validated data # COMMAND ---------- # MAGIC %md -# MAGIC Next step: [Implement a CDC pipeline for multiple tables]($./02-CDC-CDF-full-multi-tables) +# MAGIC ## Data Sharing and Datamesh Organization +# MAGIC +# MAGIC +# MAGIC +# MAGIC ### Key Benefits: +# MAGIC - 🔄 **Change Tracking**: Track all INSERT/UPDATE/DELETE operations from any Delta table +# MAGIC - 📡 **Incremental Processing**: Subscribe to table modifications as incremental processes +# MAGIC - 🏗️ **Data Mesh Ready**: Each mesh can publish tables, others can subscribe to changes +# MAGIC - 🛡️ **GDPR Compliance**: Propagate changes (e.g., GDPR DELETE) across data meshes # COMMAND ---------- -# DBTITLE 1,Make sure we stop all actives streams -DBDemos.stop_all_streams() +# MAGIC %md +# MAGIC ## Next Steps +# MAGIC +# MAGIC ### Continue Your CDC Journey: +# MAGIC - 🔗 **[Multi-Table CDC Pipeline]($./02-CDC-CDF-full-multi-tables)**: Scale to multiple tables +# MAGIC - 🏗️ **[Delta Live Tables]($./dlt-cdc)**: Simplified CDC with `APPLY CHANGES` +# MAGIC - 📚 **[Delta Lake Demo]($./delta-lake)**: Deep dive into Delta Lake features +# MAGIC - 🚀 **[Auto Loader Demo]($./auto-loader)**: Advanced file ingestion patterns \ No newline at end of file diff --git a/product_demos/cdc-pipeline/02-CDC-CDF-full-multi-tables.py b/product_demos/cdc-pipeline/02-CDC-CDF-full-multi-tables.py index 6b6614cb..8a3b4229 100644 --- a/product_demos/cdc-pipeline/02-CDC-CDF-full-multi-tables.py +++ b/product_demos/cdc-pipeline/02-CDC-CDF-full-multi-tables.py @@ -1,20 +1,34 @@ # Databricks notebook source # MAGIC %md # MAGIC -# MAGIC # Full demo: Change Data Capture on multiple tables -# MAGIC ## Use-case: Synchronize all your ELT tables with your Lakehouse +# MAGIC # Multi-Table CDC Pipeline Demo: Change Data Capture with Serverless Compute +# MAGIC ## Step-by-Step Guide to Building a Scalable Multi-Table CDC Pipeline # MAGIC -# MAGIC We previously saw how to synchronize a single table. However, real use-case typically includes multiple tables that we need to ingest and synch. +# MAGIC This demo shows you how to build a **multi-table Change Data Capture (CDC)** pipeline using **Databricks Serverless Compute** for cost-effective, auto-scaling data processing. # MAGIC -# MAGIC These tables are stored on different folder having the following layout: +# MAGIC ### What You'll Learn: +# MAGIC 1. **🔄 Step 1**: Set up multi-table CDC data simulation +# MAGIC 2. **🥉 Step 2**: Build parallel Bronze layers with Auto Loader +# MAGIC 3. **🥈 Step 3**: Create parallel Silver layers with MERGE operations +# MAGIC 4. **🚀 Step 4**: Implement Gold layer with Change Data Feed (CDF) +# MAGIC 5. **📊 Step 5**: Test Continuous multi-table CDC Data processing # MAGIC -# MAGIC # MAGIC -# MAGIC **A note on Delta Live Table**:
-# MAGIC *Delta Live Table has been designed to simplify this process and handle concurrent execution properly, without having you to start multiple stream in parallel.*
-# MAGIC *We strongly advise to have a look at the DLT CDC demo to simplify such pipeline implementation: `dbdemos.instal('dlt-cdc')`* # MAGIC -# MAGIC In this notebook, we'll see how this can be done using Python & standard streaming APIs (without DLT). +# MAGIC ### Key Benefits of Serverless Multi-Table CDC: +# MAGIC - 💰 **Cost-effective**: Pay only for compute time used across all tables +# MAGIC - 🚀 **Auto-scaling**: Automatically scales based on total workload +# MAGIC - ⚡ **Parallel Processing**: Process multiple tables simultaneously +# MAGIC - 🔄 **Incremental**: Only processes new/changed data per table +# MAGIC - 📊 **Monitoring**: Track processing across all tables +# MAGIC +# MAGIC ### Prerequisites: +# MAGIC - Completed the single-table CDC demo: `01-CDC-CDF-simple-pipeline.py` +# MAGIC - Understanding of parallel processing concepts +# MAGIC +# MAGIC --- +# MAGIC +# MAGIC **💡 Alternative Approach**: For production multi-table CDC pipelines, consider using **Delta Live Tables** with `APPLY CHANGES` for simplified implementation: `dbdemos.install('dlt-cdc')` # MAGIC # MAGIC # MAGIC @@ -25,82 +39,346 @@ # COMMAND ---------- +# DBTITLE 1,Import Required Functions +from pyspark.sql.functions import current_timestamp, col + +# COMMAND ---------- + # MAGIC %md -# MAGIC ## Running the streams in parallel +# MAGIC ## 📋 Multi-Table CDC Pipeline Architecture Overview +# MAGIC +# MAGIC Here's the complete multi-table CDC pipeline we'll build using **Serverless Compute**: # MAGIC -# MAGIC Each table will be save as a distinct table, using a distinct Spark Structured Streaming strem. +# MAGIC # MAGIC -# MAGIC To implement an efficient pipeline, we should process multiple streams at the same time. To do that, we'll use a ThreadPoolExecutor and start multiple thread, each of them processing and waiting for a stream. +# MAGIC ### Pipeline Flow: +# MAGIC 1. **📥 Data Sources**: Multiple CDC streams from different tables +# MAGIC 2. **🥉 Bronze Layers**: Parallel raw data ingestion with Auto Loader +# MAGIC 3. **🥈 Silver Layers**: Parallel data cleaning and deduplication +# MAGIC 4. **📊 Analytics**: Real-time insights across all tables # MAGIC -# MAGIC We're using Trigger Once to refresh all the tables once and then shutdown the cluster, typically every hour. For lower latencies we can keep the streams running (depending of the number of tables & cluster size), or keep the Trigger Once but loop forever. +# MAGIC ### Key Serverless Benefits: +# MAGIC - 💰 **Cost Efficiency**: Pay only for actual compute time used +# MAGIC - 🚀 **Auto-scaling**: Serverless automatically scales resources based on workload +# MAGIC - ⚡ **Parallel Processing**: Process multiple tables simultaneously +# MAGIC - 🔄 **Batch Processing**: Process all available data efficiently without continuous resource usage # MAGIC -# MAGIC *Note that for a real workload the exact number of streams depends of the total number of tables, table sizes and cluster size. We can also use several clusters to split the load if required* +# MAGIC **💡 Note**: For scheduled processing (e.g., hourly), trigger this notebook via Databricks Jobs or Workflows. # COMMAND ---------- # MAGIC %md -# MAGIC ## Schema evolution -# MAGIC -# MAGIC By organizing the raw incoming cdc files with 1 folder by table, we can easily iterate over the folders and pickup any new tables without modification. -# MAGIC -# MAGIC Schema evolution will be handled my the Autoloader and Delta `mergeSchema` option at the bronze layer. Schema evolution for MERGE (Silver Layer) are supported using `spark.databricks.delta.schema.autoMerge.enabled` -# MAGIC -# MAGIC Using these options, we'll be able to capture new tables and table schema evolution without having to change our code. +# MAGIC ## 🔄 Step 1: Set up multi-table CDC data simulation # MAGIC -# MAGIC *Note: that autoloader will trigger an error in a stream if a schema change happens, and will automatically recover during the next run. See Autoloader demo for a complete example* -# MAGIC -# MAGIC *Note: another common pattern is to redirect all the CDC events to a single message queue (the table name being a message attribute), and then dispatch the message in different Silver Tables* # COMMAND ---------- -# DBTITLE 1,Let's explore our raw cdc data. We have 2 tables we want to sync (transactions and users) +# MAGIC %md +# MAGIC ### Step 1.1: Explore Multi-Table CDC Data Structure + +# COMMAND ---------- + +print("🔍 Exploring our multi-table CDC data structure...") +print("We have 2 tables we want to sync: transactions and users") base_folder = f"{raw_data_location}/cdc" display(dbutils.fs.ls(base_folder)) # COMMAND ---------- -# MAGIC %md ## Silver and bronze transformations +# MAGIC %md +# MAGIC ### Step 1.2: Set Up Data Simulation +# MAGIC +# MAGIC To demonstrate serverless processing of multiple CDC streams simultaneously, we'll create data generators for multiple tables that simulate incoming CDC events every 60 seconds. +# MAGIC +# MAGIC ### Why This Matters: +# MAGIC - 🚀 **Parallel Processing**: Shows how serverless handles multiple streams simultaneously +# MAGIC - 💰 **Cost Efficiency**: Demonstrates auto-scaling for varying workloads +# MAGIC - 🔄 **Real-world Simulation**: Mimics multi-table CDC scenarios +# MAGIC - 📊 **Monitoring**: Enables cross-table processing visualization + +# COMMAND ---------- + +# DBTITLE 1,🎯 Step 1.2: Multi-Table CDC Data Generator Implementation +import threading +import time +import random +from datetime import datetime +import pandas as pd + +# Global variable to control the data generators +generators_running = False + +def generate_user_cdc_record(operation_type="UPDATE", user_id=None): + """Generate a single user CDC record""" + if user_id is None: + user_id = random.randint(1, 500) + + operations = { + "INSERT": { + "id": user_id, + "name": f"user_{user_id}_{random.randint(1,99)}", + "email": f"user{user_id}@company{random.randint(1,10)}.com", + "address": f"Address_{random.randint(1,999)} Street", + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "INSERT" + }, + "UPDATE": { + "id": user_id, + "name": f"updated_user_{user_id}", + "email": f"updated.user{user_id}@newcompany{random.randint(1,5)}.com", + "address": f"Updated_Address_{random.randint(1,999)} Avenue", + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "UPDATE" + }, + "DELETE": { + "id": user_id, + "name": None, + "email": None, + "address": None, + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "DELETE" + } + } + return operations[operation_type] + +def generate_transaction_cdc_record(operation_type="INSERT", transaction_id=None): + """Generate a single transaction CDC record""" + if transaction_id is None: + transaction_id = random.randint(1000, 9999) + + user_id = random.randint(1, 500) # Reference to users table + + operations = { + "INSERT": { + "id": transaction_id, + "user_id": user_id, + "amount": round(random.uniform(10.0, 1000.0), 2), + "currency": random.choice(["USD", "EUR", "GBP"]), + "transaction_type": random.choice(["purchase", "refund", "transfer"]), + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "INSERT" + }, + "UPDATE": { + "id": transaction_id, + "user_id": user_id, + "amount": round(random.uniform(10.0, 1000.0), 2), + "currency": random.choice(["USD", "EUR", "GBP"]), + "transaction_type": random.choice(["purchase", "refund", "transfer", "adjustment"]), + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "UPDATE" + }, + "DELETE": { + "id": transaction_id, + "user_id": None, + "amount": None, + "currency": None, + "transaction_type": None, + "operation_date": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "operation": "DELETE" + } + } + return operations[operation_type] + +def continuous_multi_table_generator(): + """Background function that generates CDC data for multiple tables every 60 seconds""" + global generators_running + file_counter = 0 + + while generators_running: + try: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + # Generate user CDC events + user_events = [] + num_user_events = random.randint(2, 4) + for _ in range(num_user_events): + operation = random.choices( + ["INSERT", "UPDATE", "DELETE"], + weights=[40, 50, 10] + )[0] + user_events.append(generate_user_cdc_record(operation)) + + # Generate transaction CDC events + transaction_events = [] + num_transaction_events = random.randint(3, 6) + for _ in range(num_transaction_events): + operation = random.choices( + ["INSERT", "UPDATE", "DELETE"], + weights=[70, 25, 5] # More inserts for transactions + )[0] + transaction_events.append(generate_transaction_cdc_record(operation)) + + # Save user events + user_df = pd.DataFrame(user_events) + user_filename = f"users_cdc_{timestamp}_{file_counter}.csv" + user_file_path = f"{base_folder}/users/{user_filename}" + + spark_user_df = spark.createDataFrame(user_df) + spark_user_df.coalesce(1).write.mode("overwrite").option("header", "true").csv(user_file_path) + + # Save transaction events + transaction_df = pd.DataFrame(transaction_events) + transaction_filename = f"transactions_cdc_{timestamp}_{file_counter}.csv" + transaction_file_path = f"{base_folder}/transactions/{transaction_filename}" + + spark_transaction_df = spark.createDataFrame(transaction_df) + spark_transaction_df.coalesce(1).write.mode("overwrite").option("header", "true").csv(transaction_file_path) + + print(f"Generated CDC events at {datetime.now()}:") + print(f" 📁 Users: {num_user_events} events -> {user_filename}") + print(f" 📁 Transactions: {num_transaction_events} events -> {transaction_filename}") + + file_counter += 1 + + # Wait 60 seconds before next batch + time.sleep(60) + + except Exception as e: + print(f"Error in multi-table CDC generator: {e}") + time.sleep(60) + +def start_multi_table_generators(): + """Start the multi-table CDC data generators in background""" + global generators_running + if not generators_running: + generators_running = True + generator_thread = threading.Thread(target=continuous_multi_table_generator, daemon=True) + generator_thread.start() + print("🚀 Multi-Table CDC Data Generators started!") + print("📊 Users and Transactions CDC events will arrive every 60 seconds.") + print("💡 This simulates continuous multi-table CDC for serverless processing demo.") + return generator_thread + else: + print("Multi-Table CDC Generators are already running!") + return None + +def stop_multi_table_generators(): + """Stop the multi-table CDC data generators""" + global generators_running + generators_running = False + print("🛑 Multi-Table CDC Data Generators stopped.") + +# Start the data generators for continuous multi-table simulation +print("Starting multi-table CDC simulation...") +multi_table_generator_thread = start_multi_table_generators() + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## **🥉 Step 2**: Build parallel Bronze layers with Auto Loader # COMMAND ---------- -# DBTITLE 1,let's reset all checkpoints +# DBTITLE 1,🥉 Step 2.1: Reset Checkpoints dbutils.fs.rm(f"{raw_data_location}/cdc_full", True) # COMMAND ---------- -# DBTITLE 1,Bronze ingestion with autoloader +# DBTITLE 1,🥉 Step 2.2: Bronze Ingestion with Auto Loader -#Stream using the autoloader to ingest raw files and load them in a delta table +# Stream using Auto Loader to ingest raw files and load them into Delta tables with serverless compute def update_bronze_layer(path, bronze_table): - print(f"ingesting RAW cdc data for {bronze_table} and building bronze layer...") + print(f"Ingesting RAW CDC data for {bronze_table} and building bronze layer with serverless...") + + # Drop existing table if it exists to avoid schema conflicts + try: + spark.sql(f"DROP TABLE IF EXISTS {bronze_table}") + print(f"🔄 Dropped existing {bronze_table} table to avoid schema conflicts") + except: + pass + (spark.readStream .format("cloudFiles") .option("cloudFiles.format", "csv") .option("cloudFiles.schemaLocation", f"{raw_data_location}/cdc_full/schemas/{bronze_table}") .option("cloudFiles.schemaHints", "id bigint, operation_date timestamp") .option("cloudFiles.inferColumnTypes", "true") + .option("cloudFiles.useNotifications", "false") # Optimized for serverless + .option("cloudFiles.includeExistingFiles", "false") # Only new files after checkpoint + .option("cloudFiles.maxFilesPerTrigger", "10") # Process in batches for efficiency .load(path) .withColumn("file_name", col("_metadata.file_path")) + .withColumn("processing_time", current_timestamp()) # Track when processed .writeStream .option("checkpointLocation", f"{raw_data_location}/cdc_full/checkpoints/{bronze_table}") - .option("mergeSchema", "true") - #.trigger(processingTime='10 seconds') - .trigger(availableNow=True) + .option("mergeSchema", "true") # Enable schema evolution for new columns + .trigger(availableNow=True) # Process only new data since last checkpoint .table(bronze_table).awaitTermination()) # COMMAND ---------- -# DBTITLE 1,Silver step: materialize tables with MERGE based on CDC events -#Stream incrementally loading new data from the bronze CDC table and merging them in the Silver table +# MAGIC %md +# MAGIC ## **🥈 Step 3**: Create parallel Silver layers with MERGE operations + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### 3.1 Understanding CDF vs Non-CDF Processing in Multi-Table Scenarios +# MAGIC +# MAGIC **🔍 Key Difference**: CDF only processes **actual changes** per table, while non-CDF processes **all data** across all tables. +# MAGIC +# MAGIC #### **Non-CDF Multi-Table Approach (Inefficient)**: +# MAGIC - 📊 **Processes**: Entire tables every time +# MAGIC - 💰 **Cost**: Very High - reprocesses unchanged data across all tables +# MAGIC - ⏱️ **Time**: Slow - scans all records in all tables +# MAGIC - 🔄 **Example**: If you have 5 tables with 1M records each, processes all 5M even for 1 change in 1 table +# MAGIC +# MAGIC #### **CDF Multi-Table Approach (Efficient)**: +# MAGIC - 📊 **Processes**: Only changed records per table +# MAGIC - 💰 **Cost**: Low - only pays for actual changes per table +# MAGIC - ⏱️ **Time**: Fast - processes only deltas per table +# MAGIC - 🔄 **Example**: If you have 5 tables with 1M records each but only 1 table has 5 changes, processes only 5 records +# MAGIC +# MAGIC **💡 Multi-Table CDF Benefits**: Up to 99.9%+ reduction in processing volume for incremental changes across multiple tables! +# MAGIC +# MAGIC ### 3.2 Silver Layer with MERGE Operations +# MAGIC + +# COMMAND ---------- + +# Stream incrementally loading new data from the bronze CDC table and merging them in the Silver table +# This function demonstrates CDF efficiency by processing only changed records per table def update_silver_layer(bronze_table, silver_table): - print(f"ingesting {bronze_table} update and materializing silver layer using a MERGE statement...") - #First create the silver table if it doesn't exists: + print(f"🔄 Processing {bronze_table} updates with CDF efficiency...") + + # Get total records in bronze table to show processing volume + try: + total_bronze_records = spark.sql(f"SELECT COUNT(*) as count FROM {bronze_table}").collect()[0]['count'] + print(f" 📊 Total records in {bronze_table}: {total_bronze_records:,}") + except: + total_bronze_records = 0 + print(f" 📊 Total records in {bronze_table}: {total_bronze_records:,}") + + # First create the silver table if it doesn't exist with optimized properties: if not spark.catalog.tableExists(silver_table): - print(f"Table {silver_table} doesn't exist, creating it using the same schema as the bronze one...") + print(f" 🏗️ Creating {silver_table} with optimized properties...") + # Create table with sample schema and then optimize properties spark.read.table(bronze_table).drop("operation", "operation_date", "_rescued_data", "file_name").write.saveAsTable(silver_table) + # Add optimized properties for serverless and performance + spark.sql(f""" + ALTER TABLE {silver_table} SET TBLPROPERTIES ( + delta.enableChangeDataFeed = true, + delta.autoOptimize.optimizeWrite = true, + delta.autoOptimize.autoCompact = true, + delta.targetFileSize = '128MB', + delta.tuneFileSizesForRewrites = true + ) + """) + + # Process only new records since last checkpoint (CDF efficiency) + print(f" 🔄 Processing only new records from {bronze_table}...") #for each batch / incremental update from the raw cdc table, we'll run a MERGE on the silver table def merge_stream(updates, i): + records_in_batch = updates.count() + print(f" 📊 Batch {i}: Processing {records_in_batch:,} records") + + if records_in_batch > 0 and total_bronze_records > 0: + # Show processing efficiency + efficiency = ((total_bronze_records - records_in_batch) / total_bronze_records * 100) + print(f" 💰 Processing efficiency: {efficiency:.1f}% reduction vs full table scan") + print(f" ⚡ Speed improvement: {total_bronze_records / max(records_in_batch, 1):.1f}x faster") + #First we need to deduplicate based on the id and take the most recent update windowSpec = Window.partitionBy("id").orderBy(col("operation_date").desc()) #Select only the first value @@ -116,18 +394,38 @@ def merge_stream(updates, i): .whenNotMatchedInsert("updates.operation != 'DELETE'", values=columns_to_update) \ .execute() + print(f" ✅ Batch {i} completed - processed {records_in_batch:,} records efficiently") + + print(f"🚀 Starting {silver_table} processing with CDF efficiency...") (spark.readStream .table(bronze_table) .writeStream .foreachBatch(merge_stream) .option("checkpointLocation", f"{raw_data_location}/cdc_full/checkpoints/{silver_table}") - #.trigger(processingTime='10 seconds') - .trigger(availableNow=True) + .option("mergeSchema", "true") # Enable schema evolution for silver layer + .trigger(availableNow=True) # Process only new data since last checkpoint .start().awaitTermination()) # COMMAND ---------- -# MAGIC %md ## Starting all the streams +# MAGIC %md +# MAGIC ### 3.3 Multi-Table CDF Processing Volume Summary +# MAGIC +# MAGIC **🎯 What We Just Demonstrated**: +# MAGIC - **CDF Processing**: Only processed actual changes per table +# MAGIC - **Volume Efficiency**: Dramatically reduced processing volume across multiple tables +# MAGIC - **Cost Savings**: Significant reduction in compute costs per table +# MAGIC - **Performance**: Much faster processing times per table +# MAGIC +# MAGIC **📊 Key Metrics Per Table**: +# MAGIC - **Total Bronze Records**: Shows full table size per table +# MAGIC - **CDF Records Processed**: Shows only changed records per table +# MAGIC - **Efficiency Gain**: Percentage reduction in processing volume per table +# MAGIC - **Speed Improvement**: Multiplier for processing speed per table +# MAGIC +# MAGIC **💡 Multi-Table Impact**: In production, this can mean processing 1,000 records across 5 tables instead of 5,000,000 records for incremental updates! +# MAGIC +# MAGIC ### 3.4 Starting all the streams # MAGIC # MAGIC We can now iterate over the folders to start the bronze & silver streams for each table. @@ -138,28 +436,54 @@ def merge_stream(updates, i): from delta.tables import * def refresh_cdc_table(table): + """ + Process a single CDC table using serverless compute. + Updates both bronze and silver layers with optimized settings. + """ try: - #update the bronze table + # Update the bronze table using Auto Loader with serverless optimization bronze_table = f'bronze_{table}' + print(f"Processing table: {table} -> {bronze_table}") update_bronze_layer(f"{base_folder}/{table}", bronze_table) - #then refresh the silver layer + # Then refresh the silver layer with MERGE operations silver_table = f'silver_{table}' + print(f"Materializing silver table: {silver_table}") update_silver_layer(bronze_table, silver_table) + + print(f"Successfully processed table: {table}") except Exception as e: - #prod workload should properly process errors - print(f"couldn't properly process {bronze_table}") - raise e + # Production workloads should implement comprehensive error handling + error_msg = f"Failed to process table {table}: {str(e)}" + print(error_msg) + # In production, consider: + # - Logging to external monitoring systems + # - Sending alerts/notifications + # - Continuing with other tables vs stopping entire pipeline + raise Exception(error_msg) from e -#Enable Schema evolution during merges (to capture new columns) -#spark.conf.set("spark.databricks.delta.schema.autoMerge.enabled", "true") +# Schema evolution is handled automatically by: +# - Auto Loader with mergeSchema=true option +# - Delta table mergeSchema=true in writeStream operations +# - No additional configuration needed for modern Databricks Runtime -#iterate over all the tables folders +# Iterate over all table folders and process them in parallel using serverless compute tables = [table_path.name[:-1] for table_path in dbutils.fs.ls(base_folder)] -#Let's start 3 CDC flow at the same time in 3 different thread to speed up ingestion -with ThreadPoolExecutor(max_workers=3) as executor: +print(f"Found {len(tables)} tables to process: {tables}") + +# Process multiple CDC flows simultaneously using ThreadPoolExecutor +# Serverless compute automatically scales resources based on workload +max_parallel_tables = min(len(tables), 3) # Adjust based on your data volume and processing requirements +print(f"Processing {max_parallel_tables} tables in parallel with serverless compute...") + +with ThreadPoolExecutor(max_workers=max_parallel_tables) as executor: deque(executor.map(refresh_cdc_table, tables)) - print(f"Database refreshed!") + print(f"Successfully refreshed all {len(tables)} tables using serverless compute!") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### 3.3 Check the Resulting Silver Tables # COMMAND ---------- @@ -178,15 +502,280 @@ def refresh_cdc_table(table): # MAGIC %md # MAGIC ## What's next # MAGIC -# MAGIC All our silver tables are now materialized using the CDC events! We can then work extra transformation (gold layer) based on your business requirement. +# MAGIC All our silver tables are now materialized using CDC events with **Serverless Compute**! You can now build additional transformations (gold layer) based on your business requirements. +# MAGIC +# MAGIC ### Production readiness with Serverless +# MAGIC +# MAGIC **Error Handling Strategies**: +# MAGIC - Capture and handle exceptions in each stream properly +# MAGIC - Send notifications when a table encounters errors while continuing to process others +# MAGIC - Define table processing priorities and dependencies +# MAGIC - Use Databricks Jobs/Workflows for orchestration and monitoring +# MAGIC +# MAGIC **Serverless Production Benefits**: +# MAGIC - **Cost Optimization**: Pay only for actual processing time +# MAGIC - **Auto-scaling**: Automatically scales based on data volume +# MAGIC - **Reliability**: Built-in fault tolerance and automatic restarts +# MAGIC - **Monitoring**: Integrated with Databricks monitoring and alerting +# MAGIC +# MAGIC **Scheduling Options**: +# MAGIC - Use Databricks Jobs to schedule this notebook regularly (hourly, daily) +# MAGIC - Trigger via external orchestration tools (Apache Airflow, etc.) +# MAGIC - Event-driven execution using file arrival notifications +# MAGIC +# MAGIC ### Delta Live Tables +# MAGIC To simplify these operations & error handling even further, we strongly recommend running your CDC pipelines using Delta Live Tables: `dbdemos.install('delta-live-table')` +# MAGIC +# MAGIC DLT provides native CDC support with `APPLY CHANGES` and automatic error handling, monitoring, and scaling. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 5. Test Continuous Multi-Table Serverless CDC Processing # MAGIC -# MAGIC ### Production readiness -# MAGIC Error and exception in each stream should be properly captured. Multiple strategy exist: send a notification when a table has some error and continue processing the others, stop the entire job, define table "priorities" etc. +# MAGIC With multiple data generators running, we can demonstrate how serverless compute handles continuous multi-table CDC processing efficiently and cost-effectively. The pipeline processes **only newly arrived data** across all tables. # MAGIC -# MAGIC ### Delta Live Table -# MAGIC To simplify these operations & error handling, we strongly advise you to run your CDC pipelines on top of Delta Live Table: `dbdemos.install('delta-live-table')` +# MAGIC **Multi-Table Incremental Processing:** +# MAGIC - ✅ **Per-Table Checkpoints**: Each table tracks its own processing progress +# MAGIC - ✅ **Parallel Incremental Processing**: Multiple tables process only new data simultaneously +# MAGIC - ✅ **Independent Scaling**: Each table scales based on its own data volume +# MAGIC - ✅ **No Cross-Table Reprocessing**: Changes in one table don't affect others +# MAGIC - ✅ **Efficient Resource Usage**: Pay only for actual new data processing # COMMAND ---------- -# DBTITLE 1,Make sure we stop all actives streams +# DBTITLE 1,🚀 Step 4.1: Multi-Table Pipeline Trigger Function +def trigger_multi_table_cdc_pipeline(): + """ + Trigger all multi-table CDC streams to process new data with serverless compute. + This processes all tables in parallel for maximum efficiency. + """ + print(f"🔄 Triggering multi-table CDC pipeline at {datetime.now()}") + + # Enable automatic schema merging for MERGE operations across all tables + # Schema evolution is handled automatically by mergeSchema=true in writeStream operations + + # Get all table folders + tables = [table_path.name[:-1] for table_path in dbutils.fs.ls(base_folder)] + print(f"📊 Processing {len(tables)} tables: {tables}") + + # Process all tables in parallel using ThreadPoolExecutor + max_parallel_tables = min(len(tables), 3) + print(f"⚡ Processing {max_parallel_tables} tables in parallel with serverless compute...") + + start_time = datetime.now() + + with ThreadPoolExecutor(max_workers=max_parallel_tables) as executor: + deque(executor.map(refresh_cdc_table, tables)) + + end_time = datetime.now() + processing_time = (end_time - start_time).total_seconds() + + print(f"✅ Multi-table CDC pipeline completed in {processing_time:.2f} seconds") + return processing_time + +# COMMAND ---------- + +# DBTITLE 1,🚀 Step 4: Complete Multi-Table CDC Pipeline Demo +print("🎯 Running multi-table serverless CDC processing demonstration...") +print("💡 In production, schedule this via Databricks Jobs/Workflows") + +# Give generators time to create files for both tables +print("⏳ Waiting 65 seconds for multi-table data generators to create new files...") +time.sleep(65) + +# Process all tables and measure performance +start_time = datetime.now() +processing_time = trigger_multi_table_cdc_pipeline() +total_time = (datetime.now() - start_time).total_seconds() + +print(f"\n📈 Performance Metrics:") +print(f"🔹 Total processing time: {total_time:.2f} seconds") +print(f"🔹 Parallel execution efficiency: {(processing_time/total_time)*100:.1f}%") + +# Show results with multi-table growth monitoring +print("\n📊 Monitoring multi-table growth over time...") +print("💡 Watch how serverless compute handles growing data across multiple tables") + +# Function to get all table sizes +def get_all_table_sizes(): + sizes = {} + tables = ["users", "transactions"] + + for table in tables: + bronze_table = f"bronze_{table}" + silver_table = f"silver_{table}" + + try: + sizes[f"{table}_bronze"] = spark.sql(f"SELECT COUNT(*) as count FROM {bronze_table}").collect()[0]['count'] + except: + sizes[f"{table}_bronze"] = 0 + + try: + sizes[f"{table}_silver"] = spark.sql(f"SELECT COUNT(*) as count FROM {silver_table}").collect()[0]['count'] + except: + sizes[f"{table}_silver"] = 0 + + return sizes + +# Monitor multi-table growth over multiple iterations +print("🔍 Multi-Table Growth Monitoring:") +print("=" * 80) + +for iteration in range(1, 4): # Monitor 3 iterations + print(f"\n📈 Iteration {iteration} - {datetime.now().strftime('%H:%M:%S')}") + + # Get current sizes + sizes = get_all_table_sizes() + + print("🥉 Bronze Tables (Raw CDC):") + print(f" 👥 Users: {sizes['users_bronze']:,} records") + print(f" 💳 Transactions: {sizes['transactions_bronze']:,} records") + print(f" 📊 Total Bronze: {sizes['users_bronze'] + sizes['transactions_bronze']:,} records") + + print("🥈 Silver Tables (Materialized):") + print(f" 👥 Users: {sizes['users_silver']:,} records") + print(f" 💳 Transactions: {sizes['transactions_silver']:,} records") + print(f" 📊 Total Silver: {sizes['users_silver'] + sizes['transactions_silver']:,} records") + + # Calculate growth if not first iteration + if iteration > 1: + users_bronze_growth = sizes['users_bronze'] - previous_sizes['users_bronze'] + users_silver_growth = sizes['users_silver'] - previous_sizes['users_silver'] + transactions_bronze_growth = sizes['transactions_bronze'] - previous_sizes['transactions_bronze'] + transactions_silver_growth = sizes['transactions_silver'] - previous_sizes['transactions_silver'] + + print(" 📊 Growth Since Last Check:") + print(f" 👥 Users: Bronze +{users_bronze_growth}, Silver +{users_silver_growth}") + print(f" 💳 Transactions: Bronze +{transactions_bronze_growth}, Silver +{transactions_silver_growth}") + + total_growth = (users_bronze_growth + users_silver_growth + + transactions_bronze_growth + transactions_silver_growth) + print(f" 🎯 Total Growth: +{total_growth} records across all tables") + + # Show recent activity details + print(" 🔍 Recent Activity:") + try: + # Users operations + users_ops = spark.sql(""" + SELECT operation, COUNT(*) as count + FROM bronze_users + GROUP BY operation + ORDER BY operation + """).collect() + users_summary = {row['operation']: row['count'] for row in users_ops} + print(f" 👥 Users Operations: {users_summary}") + + # Transactions operations + trans_ops = spark.sql(""" + SELECT operation, COUNT(*) as count + FROM bronze_transactions + GROUP BY operation + ORDER BY operation + """).collect() + trans_summary = {row['operation']: row['count'] for row in trans_ops} + print(f" 💳 Transactions Operations: {trans_summary}") + + # Show latest silver records + print(" 📝 Latest Records:") + latest_users = spark.sql(""" + SELECT id, name, email + FROM silver_users + ORDER BY id DESC + LIMIT 2 + """).collect() + if latest_users: + print(" 👥 Latest Users:") + for row in latest_users: + print(f" ID: {row['id']}, User: {row['name']}, Email: {row['email']}") + + latest_transactions = spark.sql(""" + SELECT id, amount, item_count + FROM silver_transactions + ORDER BY id DESC + LIMIT 2 + """).collect() + if latest_transactions: + print(" 💳 Latest Transactions:") + for row in latest_transactions: + print(f" ID: {row['id']}, Amount: {row['amount']}, Items: {row['item_count']}") + + except Exception as e: + print(f" ⚠️ Error showing details: {e}") + + previous_sizes = sizes + + # Wait for next iteration (except on last one) + if iteration < 3: + print(f" ⏳ Waiting 65 seconds for more multi-table CDC data...") + print(" 💰 Serverless compute: Zero cost during wait - only pay for processing!") + time.sleep(65) + + # Process new data across all tables + print(f" 🔄 Processing new multi-table data (Iteration {iteration + 1})...") + trigger_multi_table_cdc_pipeline() + +print("\n" + "=" * 80) +print("✅ Multi-table growth monitoring completed!") +print("📈 Key Multi-Table Observations:") +print(" 🔹 Multiple tables grow independently with different patterns") +print(" 🔹 Serverless compute scales automatically across all tables") +print(" 🔹 Parallel processing efficiency demonstrated") +print(" 🔹 Cost optimization: Pay only for actual multi-table processing") +print(" 🔹 Real-world enterprise CDC patterns with table relationships") + +# COMMAND ---------- + +# DBTITLE 1,📊 Step 5.1: Cleanup and Stop Generators +stop_multi_table_generators() DBDemos.stop_all_streams() + +print("🎉 Multi-table CDC demo completed!") +print("\n💰 Serverless Benefits Demonstrated:") +print("✅ Cost Optimization: Pay only for actual processing time") +print("✅ Auto-scaling: Handled varying workloads across multiple tables") +print("✅ Parallel Processing: Efficiently processed multiple CDC streams") +print("✅ Zero Infrastructure: No cluster management required") +print("✅ Fault Tolerance: Built-in error handling and recovery") + +print(f"\n🚀 Ready for production:") +print("• Schedule via Databricks Jobs/Workflows") +print("• Set up monitoring and alerting") +print("• Configure auto-scaling policies") +print("• Implement error handling strategies") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### Key Advantages: +# MAGIC - 🔄 **Parallel Processing**: Multiple tables processed simultaneously +# MAGIC - 📊 **Scalable Architecture**: Easy to add new tables to the pipeline +# MAGIC - 💰 **Cost Efficient**: Pay only for actual processing across all tables +# MAGIC - 🚀 **Auto-scaling**: Serverless handles varying workloads automatically +# MAGIC - 🛡️ **Fault Tolerance**: Isolated processing prevents cross-table failures + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ### Deployment Options: +# MAGIC - 📅 **Scheduled Jobs**: Use Databricks Jobs for automated processing +# MAGIC - 🔄 **Workflows**: Orchestrate complex multi-table pipelines +# MAGIC - 📊 **Monitoring**: Set up alerts and dashboards for all tables +# MAGIC - 🔒 **Security**: Implement proper access controls and data governance +# MAGIC - 💰 **Cost Optimization**: Monitor and optimize serverless compute usage + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## 📊 Step 8: Next Steps +# MAGIC +# MAGIC ### Continue Your CDC Journey: +# MAGIC - 🏗️ **[Delta Live Tables]($./dlt-cdc)**: Simplified multi-table CDC with `APPLY CHANGES` +# MAGIC - 📚 **[Delta Lake Demo]($./delta-lake)**: Deep dive into Delta Lake features +# MAGIC - 🚀 **[Auto Loader Demo]($./auto-loader)**: Advanced file ingestion patterns +# MAGIC +# MAGIC ### Advanced Patterns: +# MAGIC - 🔄 **Cross-Table Dependencies**: Handle table relationships and dependencies +# MAGIC - 📊 **Data Quality**: Implement validation and quality checks +# MAGIC - 🛡️ **Error Handling**: Advanced retry and recovery strategies +# MAGIC - 📈 **Performance Tuning**: Optimize for large-scale multi-table processing \ No newline at end of file diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/.gitignore b/product_demos/cdc-pipeline/test2/cdc_dabs/.gitignore new file mode 100644 index 00000000..0dab7f49 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/.gitignore @@ -0,0 +1,8 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +scratch/** +!scratch/README.md diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/__builtins__.pyi b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/__builtins__.pyi new file mode 100644 index 00000000..0edd5181 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/__builtins__.pyi @@ -0,0 +1,3 @@ +# Typings for Pylance in Visual Studio Code +# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md +from databricks.sdk.runtime import * diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/extensions.json b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/extensions.json new file mode 100644 index 00000000..5d15eba3 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "databricks.databricks", + "ms-python.vscode-pylance", + "redhat.vscode-yaml" + ] +} diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/settings.json b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/settings.json new file mode 100644 index 00000000..8ee87c30 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "python.analysis.stubPath": ".vscode", + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.analysis.extraPaths": ["src"], + "files.exclude": { + "**/*.egg-info": true, + "**/__pycache__": true, + ".pytest_cache": true, + }, +} diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/README.md b/product_demos/cdc-pipeline/test2/cdc_dabs/README.md new file mode 100644 index 00000000..921bb49f --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/README.md @@ -0,0 +1,51 @@ +# cdc_dabs + +The 'cdc_dabs' project was generated by using the default-python template. + +## Getting started + +0. Install UV: https://docs.astral.sh/uv/getting-started/installation/ + +1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +2. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +3. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + For example, the default template would deploy a job called + `[dev yourname] cdc_dabs_job` to your workspace. + You can find that job by opening your workpace and clicking on **Workflows**. + +4. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + + Note that the default job from the template has a schedule that runs every day + (defined in resources/cdc_dabs.job.yml). The schedule + is paused when deploying in development mode (see + https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). + +5. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` +6. Optionally, install the Databricks extension for Visual Studio code for local development from + https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your + virtual environment and setup Databricks Connect for running unit tests locally. + When not using these tools, consult your development environment's documentation + and/or the documentation for Databricks Connect for manually setting up your environment + (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html). + +7. For documentation on the Databricks asset bundles format used + for this project, and for CI/CD configuration, see + https://docs.databricks.com/dev-tools/bundles/index.html. diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/databricks.yml b/product_demos/cdc-pipeline/test2/cdc_dabs/databricks.yml new file mode 100644 index 00000000..49f9b70c --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/databricks.yml @@ -0,0 +1,35 @@ +# This is a Databricks asset bundle definition for cdc_dabs. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: cdc_dabs + uuid: 4258650d-ac2f-4d40-b813-f64d1ea65d7b + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + +include: + - resources/*.yml + - resources/*/*.yml + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://e2-demo-field-eng.cloud.databricks.com + + prod: + mode: production + workspace: + host: https://e2-demo-field-eng.cloud.databricks.com + # We explicitly deploy to /Workspace/Users/mohammad.khelghati@databricks.com to make sure we only have a single copy. + root_path: /Workspace/Users/mohammad.khelghati@databricks.com/.bundle/${bundle.name}/${bundle.target} + permissions: + - user_name: mohammad.khelghati@databricks.com + level: CAN_MANAGE diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/fixtures/.gitkeep b/product_demos/cdc-pipeline/test2/cdc_dabs/fixtures/.gitkeep new file mode 100644 index 00000000..fa25d274 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/fixtures/.gitkeep @@ -0,0 +1,22 @@ +# Fixtures + +This folder is reserved for fixtures, such as CSV files. + +Below is an example of how to load fixtures as a data frame: + +``` +import pandas as pd +import os + +def get_absolute_path(*relative_parts): + if 'dbutils' in globals(): + base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore + path = os.path.normpath(os.path.join(base_dir, *relative_parts)) + return path if path.startswith("/Workspace") else "/Workspace" + path + else: + return os.path.join(*relative_parts) + +csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") +df = pd.read_csv(csv_file) +display(df) +``` diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/pyproject.toml b/product_demos/cdc-pipeline/test2/cdc_dabs/pyproject.toml new file mode 100644 index 00000000..7ad5c9fe --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "cdc_dabs" +version = "0.0.1" +authors = [{ name = "mohammad.khelghati@databricks.com" }] +requires-python = ">= 3.11" + +[project.optional-dependencies] +dev = [ + "pytest", + + # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect + "databricks-dlt", + + # databricks-connect can be used to run parts of this project locally. + # See https://docs.databricks.com/dev-tools/databricks-connect.html. + # + # Note, databricks-connect is automatically installed if you're using Databricks + # extension for Visual Studio Code + # (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html). + # + # To manually install databricks-connect, uncomment the line below to install a version + # of db-connect that corresponds to the Databricks Runtime version used for this project. + # See https://docs.databricks.com/dev-tools/databricks-connect.html + # "databricks-connect>=15.4,<15.5", +] + +[tool.pytest.ini_options] +pythonpath = "src" +testpaths = [ + "tests", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/cdc_dabs"] + +[project.scripts] +main = "cdc_dabs.main:main" diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.job.yml b/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.job.yml new file mode 100644 index 00000000..6ae9796f --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.job.yml @@ -0,0 +1,45 @@ +# The main job for cdc_dabs. +resources: + jobs: + cdc_dabs_job: + name: cdc_dabs_job + + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS + + #email_notifications: + # on_failure: + # - your_email@example.com + + tasks: + - task_key: notebook_task + notebook_task: + notebook_path: ../src/notebook.ipynb + + - task_key: refresh_pipeline + depends_on: + - task_key: notebook_task + pipeline_task: + pipeline_id: ${resources.pipelines.cdc_dabs_pipeline.id} + + - task_key: main_task + depends_on: + - task_key: refresh_pipeline + environment_key: default + python_wheel_task: + package_name: cdc_dabs + entry_point: main + + # A list of task execution environment specifications that can be referenced by tasks of this job. + environments: + - environment_key: default + + # Full documentation of this spec can be found at: + # https://docs.databricks.com/api/workspace/jobs/create#environments-spec + spec: + client: "2" + dependencies: + - ../dist/*.whl diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.pipeline.yml b/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.pipeline.yml new file mode 100644 index 00000000..1f28896a --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/resources/cdc_dabs.pipeline.yml @@ -0,0 +1,14 @@ +# The main pipeline for cdc_dabs +resources: + pipelines: + cdc_dabs_pipeline: + name: cdc_dabs_pipeline + catalog: dbacademy + schema: cdc_dabs_${bundle.target} + serverless: true + libraries: + - notebook: + path: ../src/pipeline.ipynb + + configuration: + bundle.sourcePath: ${workspace.file_path}/src diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/scratch/README.md b/product_demos/cdc-pipeline/test2/cdc_dabs/scratch/README.md new file mode 100644 index 00000000..e6cfb81b --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/scratch/README.md @@ -0,0 +1,4 @@ +# scratch + +This folder is reserved for personal, exploratory notebooks. +By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/src/cdc_dabs/__init__.py b/product_demos/cdc-pipeline/test2/cdc_dabs/src/cdc_dabs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/src/cdc_dabs/main.py b/product_demos/cdc-pipeline/test2/cdc_dabs/src/cdc_dabs/main.py new file mode 100644 index 00000000..5ae344c7 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/src/cdc_dabs/main.py @@ -0,0 +1,25 @@ +from pyspark.sql import SparkSession, DataFrame + + +def get_taxis(spark: SparkSession) -> DataFrame: + return spark.read.table("samples.nyctaxi.trips") + + +# Create a new Databricks Connect session. If this fails, +# check that you have configured Databricks Connect correctly. +# See https://docs.databricks.com/dev-tools/databricks-connect.html. +def get_spark() -> SparkSession: + try: + from databricks.connect import DatabricksSession + + return DatabricksSession.builder.getOrCreate() + except ImportError: + return SparkSession.builder.getOrCreate() + + +def main(): + get_taxis(get_spark()).show(5) + + +if __name__ == "__main__": + main() diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/src/notebook.ipynb b/product_demos/cdc-pipeline/test2/cdc_dabs/src/notebook.ipynb new file mode 100644 index 00000000..ac4b0c1f --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/src/notebook.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "ee353e42-ff58-4955-9608-12865bd0950e", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Default notebook\n", + "\n", + "This default notebook is executed using Databricks Workflows as defined in resources/cdc_dabs.job.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "6bca260b-13d1-448f-8082-30b60a85c9ae", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "from cdc_dabs import main\n", + "\n", + "main.get_taxis(spark).show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "notebook", + "widgets": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/src/pipeline.ipynb b/product_demos/cdc-pipeline/test2/cdc_dabs/src/pipeline.ipynb new file mode 100644 index 00000000..d17f76a9 --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/src/pipeline.ipynb @@ -0,0 +1,90 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9a626959-61c8-4bba-84d2-2a4ecab1f7ec", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Lakeflow Declarative Pipeline\n", + "\n", + "This Lakeflow Declarative Pipeline (LDP) definition is executed using a pipeline defined in resources/cdc_dabs.pipeline.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "9198e987-5606-403d-9f6d-8f14e6a4017f", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Import DLT and src/cdc_dabs\n", + "import dlt\n", + "import sys\n", + "\n", + "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", + "from pyspark.sql.functions import expr\n", + "from cdc_dabs import main" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "3fc19dba-61fd-4a89-8f8c-24fee63bfb14", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "@dlt.view\n", + "def taxi_raw():\n", + " return main.get_taxis(spark)\n", + "\n", + "\n", + "@dlt.table\n", + "def filtered_taxis():\n", + " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "pipeline", + "widgets": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/product_demos/cdc-pipeline/test2/cdc_dabs/tests/main_test.py b/product_demos/cdc-pipeline/test2/cdc_dabs/tests/main_test.py new file mode 100644 index 00000000..6095c07a --- /dev/null +++ b/product_demos/cdc-pipeline/test2/cdc_dabs/tests/main_test.py @@ -0,0 +1,6 @@ +from cdc_dabs.main import get_taxis, get_spark + + +def test_main(): + taxis = get_taxis(get_spark()) + assert taxis.count() > 5