diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 68cd2f4be..8622b29ca 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -7016dcbf2e011459416cf408ce21143bcc4b3a25 \ No newline at end of file +a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index aea3eb9c8..4f7aa3cc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Version changelog +## [Release] Release v0.40.0 + +### API Changes: + + * Added [a.account_federation_policy](https://databricks-sdk-py.readthedocs.io/en/latest/account/account_federation_policy.html) account-level service and [a.service_principal_federation_policy](https://databricks-sdk-py.readthedocs.io/en/latest/account/service_principal_federation_policy.html) account-level service. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.ClusterAttributes`. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.ClusterDetails`. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.ClusterSpec`. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.CreateCluster`. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.EditCluster`. + * Added `is_single_node`, `kind` and `use_ml_runtime` fields for `databricks.sdk.service.compute.UpdateClusterResource`. + * Added `update_parameter_syntax` field for `databricks.sdk.service.dashboards.MigrateDashboardRequest`. + * Added `clean_rooms_notebook_task` field for `databricks.sdk.service.jobs.RunTask`. + * Added `clean_rooms_notebook_task` field for `databricks.sdk.service.jobs.SubmitTask`. + * Added `clean_rooms_notebook_task` field for `databricks.sdk.service.jobs.Task`. + * Changed `days_of_week` field for `databricks.sdk.service.pipelines.RestartWindow` to type `databricks.sdk.service.pipelines.RestartWindowDaysOfWeekList` dataclass. + +OpenAPI SHA: a6a317df8327c9b1e5cb59a03a42ffa2aabeef6d, Date: 2024-12-16 + ## [Release] Release v0.39.0 ### Bug Fixes diff --git a/databricks/sdk/__init__.py b/databricks/sdk/__init__.py index beb3fd7bb..068069f04 100755 --- a/databricks/sdk/__init__.py +++ b/databricks/sdk/__init__.py @@ -56,9 +56,11 @@ ProviderListingsAPI, ProviderPersonalizationRequestsAPI, ProviderProviderAnalyticsDashboardsAPI, ProviderProvidersAPI) from databricks.sdk.service.ml import ExperimentsAPI, ModelRegistryAPI -from databricks.sdk.service.oauth2 import (CustomAppIntegrationAPI, +from databricks.sdk.service.oauth2 import (AccountFederationPolicyAPI, + CustomAppIntegrationAPI, OAuthPublishedAppsAPI, PublishedAppIntegrationAPI, + ServicePrincipalFederationPolicyAPI, ServicePrincipalSecretsAPI) from databricks.sdk.service.pipelines import PipelinesAPI from databricks.sdk.service.provisioning import (CredentialsAPI, @@ -826,6 +828,7 @@ def __init__(self, self._credentials = CredentialsAPI(self._api_client) self._custom_app_integration = CustomAppIntegrationAPI(self._api_client) self._encryption_keys = EncryptionKeysAPI(self._api_client) + self._federation_policy = AccountFederationPolicyAPI(self._api_client) self._groups = AccountGroupsAPI(self._api_client) self._ip_access_lists = AccountIpAccessListsAPI(self._api_client) self._log_delivery = LogDeliveryAPI(self._api_client) @@ -836,6 +839,7 @@ def __init__(self, self._o_auth_published_apps = OAuthPublishedAppsAPI(self._api_client) self._private_access = PrivateAccessAPI(self._api_client) self._published_app_integration = PublishedAppIntegrationAPI(self._api_client) + self._service_principal_federation_policy = ServicePrincipalFederationPolicyAPI(self._api_client) self._service_principal_secrets = ServicePrincipalSecretsAPI(self._api_client) self._service_principals = AccountServicePrincipalsAPI(self._api_client) self._settings = AccountSettingsAPI(self._api_client) @@ -881,6 +885,11 @@ def encryption_keys(self) -> EncryptionKeysAPI: """These APIs manage encryption key configurations for this workspace (optional).""" return self._encryption_keys + @property + def federation_policy(self) -> AccountFederationPolicyAPI: + """These APIs manage account federation policies.""" + return self._federation_policy + @property def groups(self) -> AccountGroupsAPI: """Groups simplify identity management, making it easier to assign access to Databricks account, data, and other securable objects.""" @@ -931,6 +940,11 @@ def published_app_integration(self) -> PublishedAppIntegrationAPI: """These APIs enable administrators to manage published OAuth app integrations, which is required for adding/using Published OAuth App Integration like Tableau Desktop for Databricks in AWS cloud.""" return self._published_app_integration + @property + def service_principal_federation_policy(self) -> ServicePrincipalFederationPolicyAPI: + """These APIs manage service principal federation policies.""" + return self._service_principal_federation_policy + @property def service_principal_secrets(self) -> ServicePrincipalSecretsAPI: """These APIs enable administrators to manage service principal secrets.""" diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index 0798bb5b6..f1b549339 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -3704,8 +3704,8 @@ def from_dict(cls, d: Dict[str, any]) -> GenerateTemporaryTableCredentialRespons class GetBindingsSecurableType(Enum): CATALOG = 'catalog' + CREDENTIAL = 'credential' EXTERNAL_LOCATION = 'external_location' - SERVICE_CREDENTIAL = 'service_credential' STORAGE_CREDENTIAL = 'storage_credential' @@ -7067,8 +7067,8 @@ def from_dict(cls, d: Dict[str, any]) -> UpdateAssignmentResponse: class UpdateBindingsSecurableType(Enum): CATALOG = 'catalog' + CREDENTIAL = 'credential' EXTERNAL_LOCATION = 'external_location' - SERVICE_CREDENTIAL = 'service_credential' STORAGE_CREDENTIAL = 'storage_credential' diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index d8be32003..0afdb6f19 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -659,13 +659,19 @@ class ClusterAttributes: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -706,6 +712,20 @@ class ClusterAttributes: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + node_type_id: Optional[str] = None """This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or @@ -750,6 +770,12 @@ class ClusterAttributes: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -773,6 +799,8 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind.value if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.policy_id is not None: body['policy_id'] = self.policy_id if self.runtime_engine is not None: body['runtime_engine'] = self.runtime_engine.value @@ -781,6 +809,7 @@ def as_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = [v for v in self.ssh_public_keys] + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -805,6 +834,8 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.policy_id is not None: body['policy_id'] = self.policy_id if self.runtime_engine is not None: body['runtime_engine'] = self.runtime_engine @@ -813,6 +844,7 @@ def as_shallow_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = self.ssh_public_keys + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -834,6 +866,8 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterAttributes: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), + kind=_enum(d, 'kind', Kind), node_type_id=d.get('node_type_id', None), policy_id=d.get('policy_id', None), runtime_engine=_enum(d, 'runtime_engine', RuntimeEngine), @@ -842,6 +876,7 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterAttributes: spark_env_vars=d.get('spark_env_vars', None), spark_version=d.get('spark_version', None), ssh_public_keys=d.get('ssh_public_keys', None), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -948,13 +983,19 @@ class ClusterDetails: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -1015,10 +1056,24 @@ class ClusterDetails: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + jdbc_port: Optional[int] = None """Port on which Spark JDBC server is listening, in the driver nod. No service will be listeningon on this port in executor nodes.""" + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + last_restarted_time: Optional[int] = None """the timestamp that the cluster was started/restarted""" @@ -1111,6 +1166,12 @@ class ClusterDetails: """Information about why the cluster was terminated. This field only appears when the cluster is in a `TERMINATING` or `TERMINATED` state.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -1144,7 +1205,9 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node if self.jdbc_port is not None: body['jdbc_port'] = self.jdbc_port + if self.kind is not None: body['kind'] = self.kind.value if self.last_restarted_time is not None: body['last_restarted_time'] = self.last_restarted_time if self.last_state_loss_time is not None: body['last_state_loss_time'] = self.last_state_loss_time if self.node_type_id is not None: body['node_type_id'] = self.node_type_id @@ -1163,6 +1226,7 @@ def as_dict(self) -> dict: if self.state_message is not None: body['state_message'] = self.state_message if self.terminated_time is not None: body['terminated_time'] = self.terminated_time if self.termination_reason: body['termination_reason'] = self.termination_reason.as_dict() + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -1197,7 +1261,9 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node if self.jdbc_port is not None: body['jdbc_port'] = self.jdbc_port + if self.kind is not None: body['kind'] = self.kind if self.last_restarted_time is not None: body['last_restarted_time'] = self.last_restarted_time if self.last_state_loss_time is not None: body['last_state_loss_time'] = self.last_state_loss_time if self.node_type_id is not None: body['node_type_id'] = self.node_type_id @@ -1216,6 +1282,7 @@ def as_shallow_dict(self) -> dict: if self.state_message is not None: body['state_message'] = self.state_message if self.terminated_time is not None: body['terminated_time'] = self.terminated_time if self.termination_reason: body['termination_reason'] = self.termination_reason + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -1247,7 +1314,9 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterDetails: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), jdbc_port=d.get('jdbc_port', None), + kind=_enum(d, 'kind', Kind), last_restarted_time=d.get('last_restarted_time', None), last_state_loss_time=d.get('last_state_loss_time', None), node_type_id=d.get('node_type_id', None), @@ -1266,6 +1335,7 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterDetails: state_message=d.get('state_message', None), terminated_time=d.get('terminated_time', None), termination_reason=_from_dict(d, 'termination_reason', TerminationReason), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -1870,13 +1940,19 @@ class ClusterSpec: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -1917,6 +1993,20 @@ class ClusterSpec: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + node_type_id: Optional[str] = None """This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or @@ -1975,6 +2065,12 @@ class ClusterSpec: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -2001,6 +2097,8 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind.value if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -2010,6 +2108,7 @@ def as_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = [v for v in self.ssh_public_keys] + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -2037,6 +2136,8 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -2046,6 +2147,7 @@ def as_shallow_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = self.ssh_public_keys + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -2069,6 +2171,8 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterSpec: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), + kind=_enum(d, 'kind', Kind), node_type_id=d.get('node_type_id', None), num_workers=d.get('num_workers', None), policy_id=d.get('policy_id', None), @@ -2078,6 +2182,7 @@ def from_dict(cls, d: Dict[str, any]) -> ClusterSpec: spark_env_vars=d.get('spark_env_vars', None), spark_version=d.get('spark_version', None), ssh_public_keys=d.get('ssh_public_keys', None), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -2251,13 +2356,19 @@ class CreateCluster: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -2298,6 +2409,20 @@ class CreateCluster: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + node_type_id: Optional[str] = None """This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or @@ -2352,6 +2477,12 @@ class CreateCluster: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -2379,6 +2510,8 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind.value if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -2388,6 +2521,7 @@ def as_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = [v for v in self.ssh_public_keys] + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -2416,6 +2550,8 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -2425,6 +2561,7 @@ def as_shallow_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = self.ssh_public_keys + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -2449,6 +2586,8 @@ def from_dict(cls, d: Dict[str, any]) -> CreateCluster: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), + kind=_enum(d, 'kind', Kind), node_type_id=d.get('node_type_id', None), num_workers=d.get('num_workers', None), policy_id=d.get('policy_id', None), @@ -2458,6 +2597,7 @@ def from_dict(cls, d: Dict[str, any]) -> CreateCluster: spark_env_vars=d.get('spark_env_vars', None), spark_version=d.get('spark_version', None), ssh_public_keys=d.get('ssh_public_keys', None), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -2848,13 +2988,19 @@ class DataPlaneEventDetailsEventType(Enum): class DataSecurityMode(Enum): """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -2865,6 +3011,9 @@ class DataSecurityMode(Enum): Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled.""" + DATA_SECURITY_MODE_AUTO = 'DATA_SECURITY_MODE_AUTO' + DATA_SECURITY_MODE_DEDICATED = 'DATA_SECURITY_MODE_DEDICATED' + DATA_SECURITY_MODE_STANDARD = 'DATA_SECURITY_MODE_STANDARD' LEGACY_PASSTHROUGH = 'LEGACY_PASSTHROUGH' LEGACY_SINGLE_USER = 'LEGACY_SINGLE_USER' LEGACY_SINGLE_USER_STANDARD = 'LEGACY_SINGLE_USER_STANDARD' @@ -3306,13 +3455,19 @@ class EditCluster: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -3353,6 +3508,20 @@ class EditCluster: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + node_type_id: Optional[str] = None """This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or @@ -3407,6 +3576,12 @@ class EditCluster: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -3434,6 +3609,8 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind.value if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -3443,6 +3620,7 @@ def as_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = [v for v in self.ssh_public_keys] + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -3471,6 +3649,8 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -3480,6 +3660,7 @@ def as_shallow_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = self.ssh_public_keys + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -3504,6 +3685,8 @@ def from_dict(cls, d: Dict[str, any]) -> EditCluster: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), + kind=_enum(d, 'kind', Kind), node_type_id=d.get('node_type_id', None), num_workers=d.get('num_workers', None), policy_id=d.get('policy_id', None), @@ -3513,6 +3696,7 @@ def from_dict(cls, d: Dict[str, any]) -> EditCluster: spark_env_vars=d.get('spark_env_vars', None), spark_version=d.get('spark_version', None), ssh_public_keys=d.get('ssh_public_keys', None), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -5642,6 +5826,17 @@ def from_dict(cls, d: Dict[str, any]) -> InstanceProfile: is_meta_instance_profile=d.get('is_meta_instance_profile', None)) +class Kind(Enum): + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + + CLASSIC_PREVIEW = 'CLASSIC_PREVIEW' + + class Language(Enum): PYTHON = 'python' @@ -7560,13 +7755,19 @@ class UpdateClusterResource: data_security_mode: Optional[DataSecurityMode] = None """Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features - are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively - used by a single user specified in `single_user_name`. Most programming languages, cluster - features and data governance features are available in this mode. * `USER_ISOLATION`: A secure - cluster that can be shared by multiple users. Cluster users are fully isolated so that they - cannot see each other's data and credentials. Most data governance features are supported in - this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: + Alias for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for + multiple users sharing the cluster. Data governance features are not available in this mode. * + `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in + `single_user_name`. Most programming languages, cluster features and data governance features + are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple + users. Cluster users are fully isolated so that they cannot see each other's data and + credentials. Most data governance features are supported in this mode. But programming languages + and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -7607,6 +7808,20 @@ class UpdateClusterResource: instance_pool_id: Optional[str] = None """The optional ID of the instance pool to which the cluster belongs.""" + is_single_node: Optional[bool] = None + """This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, + `spark_conf`, and `num_workers`""" + + kind: Optional[Kind] = None + """The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = + CLASSIC_PREVIEW`.""" + node_type_id: Optional[str] = None """This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or @@ -7665,6 +7880,12 @@ class UpdateClusterResource: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + use_ml_runtime: Optional[bool] = None + """This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not.""" + workload_type: Optional[WorkloadType] = None def as_dict(self) -> dict: @@ -7689,6 +7910,8 @@ def as_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes.as_dict() if self.init_scripts: body['init_scripts'] = [v.as_dict() for v in self.init_scripts] if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind.value if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -7698,6 +7921,7 @@ def as_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = [v for v in self.ssh_public_keys] + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type.as_dict() return body @@ -7723,6 +7947,8 @@ def as_shallow_dict(self) -> dict: if self.gcp_attributes: body['gcp_attributes'] = self.gcp_attributes if self.init_scripts: body['init_scripts'] = self.init_scripts if self.instance_pool_id is not None: body['instance_pool_id'] = self.instance_pool_id + if self.is_single_node is not None: body['is_single_node'] = self.is_single_node + if self.kind is not None: body['kind'] = self.kind if self.node_type_id is not None: body['node_type_id'] = self.node_type_id if self.num_workers is not None: body['num_workers'] = self.num_workers if self.policy_id is not None: body['policy_id'] = self.policy_id @@ -7732,6 +7958,7 @@ def as_shallow_dict(self) -> dict: if self.spark_env_vars: body['spark_env_vars'] = self.spark_env_vars if self.spark_version is not None: body['spark_version'] = self.spark_version if self.ssh_public_keys: body['ssh_public_keys'] = self.ssh_public_keys + if self.use_ml_runtime is not None: body['use_ml_runtime'] = self.use_ml_runtime if self.workload_type: body['workload_type'] = self.workload_type return body @@ -7754,6 +7981,8 @@ def from_dict(cls, d: Dict[str, any]) -> UpdateClusterResource: gcp_attributes=_from_dict(d, 'gcp_attributes', GcpAttributes), init_scripts=_repeated_dict(d, 'init_scripts', InitScriptInfo), instance_pool_id=d.get('instance_pool_id', None), + is_single_node=d.get('is_single_node', None), + kind=_enum(d, 'kind', Kind), node_type_id=d.get('node_type_id', None), num_workers=d.get('num_workers', None), policy_id=d.get('policy_id', None), @@ -7763,6 +7992,7 @@ def from_dict(cls, d: Dict[str, any]) -> UpdateClusterResource: spark_env_vars=d.get('spark_env_vars', None), spark_version=d.get('spark_version', None), ssh_public_keys=d.get('ssh_public_keys', None), + use_ml_runtime=d.get('use_ml_runtime', None), workload_type=_from_dict(d, 'workload_type', WorkloadType)) @@ -8301,6 +8531,8 @@ def create(self, gcp_attributes: Optional[GcpAttributes] = None, init_scripts: Optional[List[InitScriptInfo]] = None, instance_pool_id: Optional[str] = None, + is_single_node: Optional[bool] = None, + kind: Optional[Kind] = None, node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, @@ -8309,6 +8541,7 @@ def create(self, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None) -> Wait[ClusterDetails]: """Create new cluster. @@ -8364,13 +8597,19 @@ def create(self, :param data_security_mode: :class:`DataSecurityMode` (optional) Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are - not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a - single user specified in `single_user_name`. Most programming languages, cluster features and data - governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be - shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data - and credentials. Most data governance features are supported in this mode. But programming languages - and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias + for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple + users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: + A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. + Most programming languages, cluster features and data governance features are available in this + mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are + fully isolated so that they cannot see each other's data and credentials. Most data governance + features are supported in this mode. But programming languages and cluster features might be + limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -8402,6 +8641,17 @@ def create(self, logs are sent to `//init_scripts`. :param instance_pool_id: str (optional) The optional ID of the instance pool to which the cluster belongs. + :param is_single_node: bool (optional) + This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, + and `num_workers` + :param kind: :class:`Kind` (optional) + The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = CLASSIC_PREVIEW`. :param node_type_id: str (optional) This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute @@ -8448,6 +8698,11 @@ def create(self, SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param use_ml_runtime: bool (optional) + This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not. :param workload_type: :class:`WorkloadType` (optional) :returns: @@ -8475,6 +8730,8 @@ def create(self, if gcp_attributes is not None: body['gcp_attributes'] = gcp_attributes.as_dict() if init_scripts is not None: body['init_scripts'] = [v.as_dict() for v in init_scripts] if instance_pool_id is not None: body['instance_pool_id'] = instance_pool_id + if is_single_node is not None: body['is_single_node'] = is_single_node + if kind is not None: body['kind'] = kind.value if node_type_id is not None: body['node_type_id'] = node_type_id if num_workers is not None: body['num_workers'] = num_workers if policy_id is not None: body['policy_id'] = policy_id @@ -8484,6 +8741,7 @@ def create(self, if spark_env_vars is not None: body['spark_env_vars'] = spark_env_vars if spark_version is not None: body['spark_version'] = spark_version if ssh_public_keys is not None: body['ssh_public_keys'] = [v for v in ssh_public_keys] + if use_ml_runtime is not None: body['use_ml_runtime'] = use_ml_runtime if workload_type is not None: body['workload_type'] = workload_type.as_dict() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } @@ -8514,6 +8772,8 @@ def create_and_wait( gcp_attributes: Optional[GcpAttributes] = None, init_scripts: Optional[List[InitScriptInfo]] = None, instance_pool_id: Optional[str] = None, + is_single_node: Optional[bool] = None, + kind: Optional[Kind] = None, node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, @@ -8522,6 +8782,7 @@ def create_and_wait( spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, timeout=timedelta(minutes=20)) -> ClusterDetails: return self.create(apply_policy_default_values=apply_policy_default_values, @@ -8542,6 +8803,8 @@ def create_and_wait( gcp_attributes=gcp_attributes, init_scripts=init_scripts, instance_pool_id=instance_pool_id, + is_single_node=is_single_node, + kind=kind, node_type_id=node_type_id, num_workers=num_workers, policy_id=policy_id, @@ -8551,6 +8814,7 @@ def create_and_wait( spark_env_vars=spark_env_vars, spark_version=spark_version, ssh_public_keys=ssh_public_keys, + use_ml_runtime=use_ml_runtime, workload_type=workload_type).result(timeout=timeout) def delete(self, cluster_id: str) -> Wait[ClusterDetails]: @@ -8600,6 +8864,8 @@ def edit(self, gcp_attributes: Optional[GcpAttributes] = None, init_scripts: Optional[List[InitScriptInfo]] = None, instance_pool_id: Optional[str] = None, + is_single_node: Optional[bool] = None, + kind: Optional[Kind] = None, node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, @@ -8608,6 +8874,7 @@ def edit(self, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None) -> Wait[ClusterDetails]: """Update cluster configuration. @@ -8663,13 +8930,19 @@ def edit(self, :param data_security_mode: :class:`DataSecurityMode` (optional) Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are - not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a - single user specified in `single_user_name`. Most programming languages, cluster features and data - governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be - shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data - and credentials. Most data governance features are supported in this mode. But programming languages - and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias + for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple + users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: + A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. + Most programming languages, cluster features and data governance features are available in this + mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are + fully isolated so that they cannot see each other's data and credentials. Most data governance + features are supported in this mode. But programming languages and cluster features might be + limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -8701,6 +8974,17 @@ def edit(self, logs are sent to `//init_scripts`. :param instance_pool_id: str (optional) The optional ID of the instance pool to which the cluster belongs. + :param is_single_node: bool (optional) + This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, + and `num_workers` + :param kind: :class:`Kind` (optional) + The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = CLASSIC_PREVIEW`. :param node_type_id: str (optional) This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute @@ -8747,6 +9031,11 @@ def edit(self, SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param use_ml_runtime: bool (optional) + This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not. :param workload_type: :class:`WorkloadType` (optional) :returns: @@ -8774,6 +9063,8 @@ def edit(self, if gcp_attributes is not None: body['gcp_attributes'] = gcp_attributes.as_dict() if init_scripts is not None: body['init_scripts'] = [v.as_dict() for v in init_scripts] if instance_pool_id is not None: body['instance_pool_id'] = instance_pool_id + if is_single_node is not None: body['is_single_node'] = is_single_node + if kind is not None: body['kind'] = kind.value if node_type_id is not None: body['node_type_id'] = node_type_id if num_workers is not None: body['num_workers'] = num_workers if policy_id is not None: body['policy_id'] = policy_id @@ -8783,6 +9074,7 @@ def edit(self, if spark_env_vars is not None: body['spark_env_vars'] = spark_env_vars if spark_version is not None: body['spark_version'] = spark_version if ssh_public_keys is not None: body['ssh_public_keys'] = [v for v in ssh_public_keys] + if use_ml_runtime is not None: body['use_ml_runtime'] = use_ml_runtime if workload_type is not None: body['workload_type'] = workload_type.as_dict() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } @@ -8813,6 +9105,8 @@ def edit_and_wait( gcp_attributes: Optional[GcpAttributes] = None, init_scripts: Optional[List[InitScriptInfo]] = None, instance_pool_id: Optional[str] = None, + is_single_node: Optional[bool] = None, + kind: Optional[Kind] = None, node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, @@ -8821,6 +9115,7 @@ def edit_and_wait( spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, timeout=timedelta(minutes=20)) -> ClusterDetails: return self.edit(apply_policy_default_values=apply_policy_default_values, @@ -8841,6 +9136,8 @@ def edit_and_wait( gcp_attributes=gcp_attributes, init_scripts=init_scripts, instance_pool_id=instance_pool_id, + is_single_node=is_single_node, + kind=kind, node_type_id=node_type_id, num_workers=num_workers, policy_id=policy_id, @@ -8850,6 +9147,7 @@ def edit_and_wait( spark_env_vars=spark_env_vars, spark_version=spark_version, ssh_public_keys=ssh_public_keys, + use_ml_runtime=use_ml_runtime, workload_type=workload_type).result(timeout=timeout) def events(self, diff --git a/databricks/sdk/service/dashboards.py b/databricks/sdk/service/dashboards.py index da908cb2d..34bd58995 100755 --- a/databricks/sdk/service/dashboards.py +++ b/databricks/sdk/service/dashboards.py @@ -711,12 +711,18 @@ class MigrateDashboardRequest: parent_path: Optional[str] = None """The workspace path of the folder to contain the migrated Lakeview dashboard.""" + update_parameter_syntax: Optional[bool] = None + """Flag to indicate if mustache parameter syntax ({{ param }}) should be auto-updated to named + syntax (:param) when converting datasets in the dashboard.""" + def as_dict(self) -> dict: """Serializes the MigrateDashboardRequest into a dictionary suitable for use as a JSON request body.""" body = {} if self.display_name is not None: body['display_name'] = self.display_name if self.parent_path is not None: body['parent_path'] = self.parent_path if self.source_dashboard_id is not None: body['source_dashboard_id'] = self.source_dashboard_id + if self.update_parameter_syntax is not None: + body['update_parameter_syntax'] = self.update_parameter_syntax return body def as_shallow_dict(self) -> dict: @@ -725,6 +731,8 @@ def as_shallow_dict(self) -> dict: if self.display_name is not None: body['display_name'] = self.display_name if self.parent_path is not None: body['parent_path'] = self.parent_path if self.source_dashboard_id is not None: body['source_dashboard_id'] = self.source_dashboard_id + if self.update_parameter_syntax is not None: + body['update_parameter_syntax'] = self.update_parameter_syntax return body @classmethod @@ -732,7 +740,8 @@ def from_dict(cls, d: Dict[str, any]) -> MigrateDashboardRequest: """Deserializes the MigrateDashboardRequest from a dictionary.""" return cls(display_name=d.get('display_name', None), parent_path=d.get('parent_path', None), - source_dashboard_id=d.get('source_dashboard_id', None)) + source_dashboard_id=d.get('source_dashboard_id', None), + update_parameter_syntax=d.get('update_parameter_syntax', None)) @dataclass @@ -1759,7 +1768,8 @@ def migrate(self, source_dashboard_id: str, *, display_name: Optional[str] = None, - parent_path: Optional[str] = None) -> Dashboard: + parent_path: Optional[str] = None, + update_parameter_syntax: Optional[bool] = None) -> Dashboard: """Migrate dashboard. Migrates a classic SQL dashboard to Lakeview. @@ -1770,6 +1780,9 @@ def migrate(self, Display name for the new Lakeview dashboard. :param parent_path: str (optional) The workspace path of the folder to contain the migrated Lakeview dashboard. + :param update_parameter_syntax: bool (optional) + Flag to indicate if mustache parameter syntax ({{ param }}) should be auto-updated to named syntax + (:param) when converting datasets in the dashboard. :returns: :class:`Dashboard` """ @@ -1777,6 +1790,7 @@ def migrate(self, if display_name is not None: body['display_name'] = display_name if parent_path is not None: body['parent_path'] = parent_path if source_dashboard_id is not None: body['source_dashboard_id'] = source_dashboard_id + if update_parameter_syntax is not None: body['update_parameter_syntax'] = update_parameter_syntax headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } res = self._api.do('POST', '/api/2.0/lakeview/dashboards/migrate', body=body, headers=headers) diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index a991c7c50..105c7cd22 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -209,7 +209,8 @@ class BaseRun: previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update.""" + triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually + restart a continuous job run.""" trigger_info: Optional[TriggerInfo] = None """Additional details about what triggered the run""" @@ -449,7 +450,7 @@ class CleanRoomTaskRunResultState(Enum): @dataclass class CleanRoomTaskRunState: - """Stores the run state of the clean room notebook V1 task.""" + """Stores the run state of the clean rooms notebook task.""" life_cycle_state: Optional[CleanRoomTaskRunLifeCycleState] = None """A value indicating the run's current lifecycle state. This field is always available in the @@ -479,6 +480,48 @@ def from_dict(cls, d: Dict[str, any]) -> CleanRoomTaskRunState: result_state=_enum(d, 'result_state', CleanRoomTaskRunResultState)) +@dataclass +class CleanRoomsNotebookTask: + clean_room_name: str + """The clean room that the notebook belongs to.""" + + notebook_name: str + """Name of the notebook being run.""" + + etag: Optional[str] = None + """Checksum to validate the freshness of the notebook resource (i.e. the notebook being run is the + latest version). It can be fetched by calling the :method:cleanroomassets/get API.""" + + notebook_base_parameters: Optional[Dict[str, str]] = None + """Base parameters to be used for the clean room notebook job.""" + + def as_dict(self) -> dict: + """Serializes the CleanRoomsNotebookTask into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.clean_room_name is not None: body['clean_room_name'] = self.clean_room_name + if self.etag is not None: body['etag'] = self.etag + if self.notebook_base_parameters: body['notebook_base_parameters'] = self.notebook_base_parameters + if self.notebook_name is not None: body['notebook_name'] = self.notebook_name + return body + + def as_shallow_dict(self) -> dict: + """Serializes the CleanRoomsNotebookTask into a shallow dictionary of its immediate attributes.""" + body = {} + if self.clean_room_name is not None: body['clean_room_name'] = self.clean_room_name + if self.etag is not None: body['etag'] = self.etag + if self.notebook_base_parameters: body['notebook_base_parameters'] = self.notebook_base_parameters + if self.notebook_name is not None: body['notebook_name'] = self.notebook_name + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> CleanRoomsNotebookTask: + """Deserializes the CleanRoomsNotebookTask from a dictionary.""" + return cls(clean_room_name=d.get('clean_room_name', None), + etag=d.get('etag', None), + notebook_base_parameters=d.get('notebook_base_parameters', None), + notebook_name=d.get('notebook_name', None)) + + @dataclass class ClusterInstance: cluster_id: Optional[str] = None @@ -2526,11 +2569,11 @@ class JobsHealthMetric(Enum): * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric - is in Private Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag - across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_SECONDS`: An - estimate of the maximum consumer delay across all streams. This metric is in Private Preview. * + is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag + across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate + of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all - streams. This metric is in Private Preview.""" + streams. This metric is in Public Preview.""" RUN_DURATION_SECONDS = 'RUN_DURATION_SECONDS' STREAMING_BACKLOG_BYTES = 'STREAMING_BACKLOG_BYTES' @@ -2552,11 +2595,11 @@ class JobsHealthRule: * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric - is in Private Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag - across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_SECONDS`: An - estimate of the maximum consumer delay across all streams. This metric is in Private Preview. * + is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag + across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate + of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all - streams. This metric is in Private Preview.""" + streams. This metric is in Public Preview.""" op: JobsHealthOperator """Specifies the operator used to compare the health metric value with the specified threshold.""" @@ -3711,7 +3754,8 @@ class Run: previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update.""" + triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually + restart a continuous job run.""" trigger_info: Optional[TriggerInfo] = None """Additional details about what triggered the run""" @@ -4653,6 +4697,11 @@ class RunTask: original attempt’s ID and an incrementing `attempt_number`. Runs are retried only until they succeed, and the maximum `attempt_number` is the same as the `max_retries` value for the job.""" + clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None + """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present. + + [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html""" + cleanup_duration: Optional[int] = None """The time in milliseconds it took to terminate the cluster and clean up any associated artifacts. The duration of a task run is the sum of the `setup_duration`, `execution_duration`, and the @@ -4820,6 +4869,8 @@ def as_dict(self) -> dict: """Serializes the RunTask into a dictionary suitable for use as a JSON request body.""" body = {} if self.attempt_number is not None: body['attempt_number'] = self.attempt_number + if self.clean_rooms_notebook_task: + body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task.as_dict() if self.cleanup_duration is not None: body['cleanup_duration'] = self.cleanup_duration if self.cluster_instance: body['cluster_instance'] = self.cluster_instance.as_dict() if self.condition_task: body['condition_task'] = self.condition_task.as_dict() @@ -4864,6 +4915,7 @@ def as_shallow_dict(self) -> dict: """Serializes the RunTask into a shallow dictionary of its immediate attributes.""" body = {} if self.attempt_number is not None: body['attempt_number'] = self.attempt_number + if self.clean_rooms_notebook_task: body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task if self.cleanup_duration is not None: body['cleanup_duration'] = self.cleanup_duration if self.cluster_instance: body['cluster_instance'] = self.cluster_instance if self.condition_task: body['condition_task'] = self.condition_task @@ -4908,6 +4960,8 @@ def as_shallow_dict(self) -> dict: def from_dict(cls, d: Dict[str, any]) -> RunTask: """Deserializes the RunTask from a dictionary.""" return cls(attempt_number=d.get('attempt_number', None), + clean_rooms_notebook_task=_from_dict(d, 'clean_rooms_notebook_task', + CleanRoomsNotebookTask), cleanup_duration=d.get('cleanup_duration', None), cluster_instance=_from_dict(d, 'cluster_instance', ClusterInstance), condition_task=_from_dict(d, 'condition_task', RunConditionTask), @@ -5753,6 +5807,11 @@ class SubmitTask: field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" + clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None + """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present. + + [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html""" + condition_task: Optional[ConditionTask] = None """The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and @@ -5857,6 +5916,8 @@ class SubmitTask: def as_dict(self) -> dict: """Serializes the SubmitTask into a dictionary suitable for use as a JSON request body.""" body = {} + if self.clean_rooms_notebook_task: + body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task.as_dict() if self.condition_task: body['condition_task'] = self.condition_task.as_dict() if self.dbt_task: body['dbt_task'] = self.dbt_task.as_dict() if self.depends_on: body['depends_on'] = [v.as_dict() for v in self.depends_on] @@ -5886,6 +5947,7 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the SubmitTask into a shallow dictionary of its immediate attributes.""" body = {} + if self.clean_rooms_notebook_task: body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task if self.condition_task: body['condition_task'] = self.condition_task if self.dbt_task: body['dbt_task'] = self.dbt_task if self.depends_on: body['depends_on'] = self.depends_on @@ -5915,7 +5977,9 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> SubmitTask: """Deserializes the SubmitTask from a dictionary.""" - return cls(condition_task=_from_dict(d, 'condition_task', ConditionTask), + return cls(clean_rooms_notebook_task=_from_dict(d, 'clean_rooms_notebook_task', + CleanRoomsNotebookTask), + condition_task=_from_dict(d, 'condition_task', ConditionTask), dbt_task=_from_dict(d, 'dbt_task', DbtTask), depends_on=_repeated_dict(d, 'depends_on', TaskDependency), description=d.get('description', None), @@ -5997,6 +6061,11 @@ class Task: field is required and must be unique within its parent job. On Update or Reset, this field is used to reference the tasks to be updated or reset.""" + clean_rooms_notebook_task: Optional[CleanRoomsNotebookTask] = None + """The task runs a [clean rooms] notebook when the `clean_rooms_notebook_task` field is present. + + [clean rooms]: https://docs.databricks.com/en/clean-rooms/index.html""" + condition_task: Optional[ConditionTask] = None """The task evaluates a condition that can be used to control the execution of other tasks when the `condition_task` field is present. The condition task does not require a cluster to execute and @@ -6126,6 +6195,8 @@ class Task: def as_dict(self) -> dict: """Serializes the Task into a dictionary suitable for use as a JSON request body.""" body = {} + if self.clean_rooms_notebook_task: + body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task.as_dict() if self.condition_task: body['condition_task'] = self.condition_task.as_dict() if self.dbt_task: body['dbt_task'] = self.dbt_task.as_dict() if self.depends_on: body['depends_on'] = [v.as_dict() for v in self.depends_on] @@ -6162,6 +6233,7 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the Task into a shallow dictionary of its immediate attributes.""" body = {} + if self.clean_rooms_notebook_task: body['clean_rooms_notebook_task'] = self.clean_rooms_notebook_task if self.condition_task: body['condition_task'] = self.condition_task if self.dbt_task: body['dbt_task'] = self.dbt_task if self.depends_on: body['depends_on'] = self.depends_on @@ -6198,7 +6270,9 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> Task: """Deserializes the Task from a dictionary.""" - return cls(condition_task=_from_dict(d, 'condition_task', ConditionTask), + return cls(clean_rooms_notebook_task=_from_dict(d, 'clean_rooms_notebook_task', + CleanRoomsNotebookTask), + condition_task=_from_dict(d, 'condition_task', ConditionTask), dbt_task=_from_dict(d, 'dbt_task', DbtTask), depends_on=_repeated_dict(d, 'depends_on', TaskDependency), description=d.get('description', None), @@ -6610,7 +6684,8 @@ class TriggerType(Enum): previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update.""" + triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually + restart a continuous job run.""" FILE_ARRIVAL = 'FILE_ARRIVAL' ONE_TIME = 'ONE_TIME' diff --git a/databricks/sdk/service/oauth2.py b/databricks/sdk/service/oauth2.py index 11a83b3ab..f7df5a25e 100755 --- a/databricks/sdk/service/oauth2.py +++ b/databricks/sdk/service/oauth2.py @@ -288,6 +288,60 @@ def from_dict(cls, d: Dict[str, any]) -> DeleteResponse: return cls() +@dataclass +class FederationPolicy: + create_time: Optional[str] = None + """Creation time of the federation policy.""" + + description: Optional[str] = None + """Description of the federation policy.""" + + name: Optional[str] = None + """Name of the federation policy. The name must contain only lowercase alphanumeric characters, + numbers, and hyphens. It must be unique within the account.""" + + oidc_policy: Optional[OidcFederationPolicy] = None + """Specifies the policy to use for validating OIDC claims in your federated tokens.""" + + uid: Optional[str] = None + """Unique, immutable id of the federation policy.""" + + update_time: Optional[str] = None + """Last update time of the federation policy.""" + + def as_dict(self) -> dict: + """Serializes the FederationPolicy into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.create_time is not None: body['create_time'] = self.create_time + if self.description is not None: body['description'] = self.description + if self.name is not None: body['name'] = self.name + if self.oidc_policy: body['oidc_policy'] = self.oidc_policy.as_dict() + if self.uid is not None: body['uid'] = self.uid + if self.update_time is not None: body['update_time'] = self.update_time + return body + + def as_shallow_dict(self) -> dict: + """Serializes the FederationPolicy into a shallow dictionary of its immediate attributes.""" + body = {} + if self.create_time is not None: body['create_time'] = self.create_time + if self.description is not None: body['description'] = self.description + if self.name is not None: body['name'] = self.name + if self.oidc_policy: body['oidc_policy'] = self.oidc_policy + if self.uid is not None: body['uid'] = self.uid + if self.update_time is not None: body['update_time'] = self.update_time + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> FederationPolicy: + """Deserializes the FederationPolicy from a dictionary.""" + return cls(create_time=d.get('create_time', None), + description=d.get('description', None), + name=d.get('name', None), + oidc_policy=_from_dict(d, 'oidc_policy', OidcFederationPolicy), + uid=d.get('uid', None), + update_time=d.get('update_time', None)) + + @dataclass class GetCustomAppIntegrationOutput: client_id: Optional[str] = None @@ -498,6 +552,33 @@ def from_dict(cls, d: Dict[str, any]) -> GetPublishedAppsOutput: next_page_token=d.get('next_page_token', None)) +@dataclass +class ListFederationPoliciesResponse: + next_page_token: Optional[str] = None + + policies: Optional[List[FederationPolicy]] = None + + def as_dict(self) -> dict: + """Serializes the ListFederationPoliciesResponse into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.next_page_token is not None: body['next_page_token'] = self.next_page_token + if self.policies: body['policies'] = [v.as_dict() for v in self.policies] + return body + + def as_shallow_dict(self) -> dict: + """Serializes the ListFederationPoliciesResponse into a shallow dictionary of its immediate attributes.""" + body = {} + if self.next_page_token is not None: body['next_page_token'] = self.next_page_token + if self.policies: body['policies'] = self.policies + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> ListFederationPoliciesResponse: + """Deserializes the ListFederationPoliciesResponse from a dictionary.""" + return cls(next_page_token=d.get('next_page_token', None), + policies=_repeated_dict(d, 'policies', FederationPolicy)) + + @dataclass class ListServicePrincipalSecretsResponse: next_page_token: Optional[str] = None @@ -527,6 +608,64 @@ def from_dict(cls, d: Dict[str, any]) -> ListServicePrincipalSecretsResponse: secrets=_repeated_dict(d, 'secrets', SecretInfo)) +@dataclass +class OidcFederationPolicy: + """Specifies the policy to use for validating OIDC claims in your federated tokens.""" + + audiences: Optional[List[str]] = None + """The allowed token audiences, as specified in the 'aud' claim of federated tokens. The audience + identifier is intended to represent the recipient of the token. Can be any non-empty string + value. As long as the audience in the token matches at least one audience in the policy, the + token is considered a match. If audiences is unspecified, defaults to your Databricks account + id.""" + + issuer: Optional[str] = None + """The required token issuer, as specified in the 'iss' claim of federated tokens.""" + + jwks_json: Optional[str] = None + """The public keys used to validate the signature of federated tokens, in JWKS format. If + unspecified (recommended), Databricks automatically fetches the public keys from your issuer’s + well known endpoint. Databricks strongly recommends relying on your issuer’s well known + endpoint for discovering public keys.""" + + subject: Optional[str] = None + """The required token subject, as specified in the subject claim of federated tokens. Must be + specified for service principal federation policies. Must not be specified for account + federation policies.""" + + subject_claim: Optional[str] = None + """The claim that contains the subject of the token. If unspecified, the default value is 'sub'.""" + + def as_dict(self) -> dict: + """Serializes the OidcFederationPolicy into a dictionary suitable for use as a JSON request body.""" + body = {} + if self.audiences: body['audiences'] = [v for v in self.audiences] + if self.issuer is not None: body['issuer'] = self.issuer + if self.jwks_json is not None: body['jwks_json'] = self.jwks_json + if self.subject is not None: body['subject'] = self.subject + if self.subject_claim is not None: body['subject_claim'] = self.subject_claim + return body + + def as_shallow_dict(self) -> dict: + """Serializes the OidcFederationPolicy into a shallow dictionary of its immediate attributes.""" + body = {} + if self.audiences: body['audiences'] = self.audiences + if self.issuer is not None: body['issuer'] = self.issuer + if self.jwks_json is not None: body['jwks_json'] = self.jwks_json + if self.subject is not None: body['subject'] = self.subject + if self.subject_claim is not None: body['subject_claim'] = self.subject_claim + return body + + @classmethod + def from_dict(cls, d: Dict[str, any]) -> OidcFederationPolicy: + """Deserializes the OidcFederationPolicy from a dictionary.""" + return cls(audiences=d.get('audiences', None), + issuer=d.get('issuer', None), + jwks_json=d.get('jwks_json', None), + subject=d.get('subject', None), + subject_claim=d.get('subject_claim', None)) + + @dataclass class PublishedAppOutput: app_id: Optional[str] = None @@ -769,6 +908,158 @@ def from_dict(cls, d: Dict[str, any]) -> UpdatePublishedAppIntegrationOutput: return cls() +class AccountFederationPolicyAPI: + """These APIs manage account federation policies. + + Account federation policies allow users and service principals in your Databricks account to securely + access Databricks APIs using tokens from your trusted identity providers (IdPs). + + With token federation, your users and service principals can exchange tokens from your IdP for Databricks + OAuth tokens, which can be used to access Databricks APIs. Token federation eliminates the need to manage + Databricks secrets, and allows you to centralize management of token issuance policies in your IdP. + Databricks token federation is typically used in combination with [SCIM], so users in your IdP are + synchronized into your Databricks account. + + Token federation is configured in your Databricks account using an account federation policy. An account + federation policy specifies: * which IdP, or issuer, your Databricks account should accept tokens from * + how to determine which Databricks user, or subject, a token is issued for + + To configure a federation policy, you provide the following: * The required token __issuer__, as specified + in the “iss” claim of your tokens. The issuer is an https URL that identifies your IdP. * The allowed + token __audiences__, as specified in the “aud” claim of your tokens. This identifier is intended to + represent the recipient of the token. As long as the audience in the token matches at least one audience + in the policy, the token is considered a match. If unspecified, the default value is your Databricks + account id. * The __subject claim__, which indicates which token claim contains the Databricks username of + the user the token was issued for. If unspecified, the default value is “sub”. * Optionally, the + public keys used to validate the signature of your tokens, in JWKS format. If unspecified (recommended), + Databricks automatically fetches the public keys from your issuer’s well known endpoint. Databricks + strongly recommends relying on your issuer’s well known endpoint for discovering public keys. + + An example federation policy is: ``` issuer: "https://idp.mycompany.com/oidc" audiences: ["databricks"] + subject_claim: "sub" ``` + + An example JWT token body that matches this policy and could be used to authenticate to Databricks as user + `username@mycompany.com` is: ``` { "iss": "https://idp.mycompany.com/oidc", "aud": "databricks", "sub": + "username@mycompany.com" } ``` + + You may also need to configure your IdP to generate tokens for your users to exchange with Databricks, if + your users do not already have the ability to generate tokens that are compatible with your federation + policy. + + You do not need to configure an OAuth application in Databricks to use token federation. + + [SCIM]: https://docs.databricks.com/admin/users-groups/scim/index.html""" + + def __init__(self, api_client): + self._api = api_client + + def create(self, + *, + policy: Optional[FederationPolicy] = None, + policy_id: Optional[str] = None) -> FederationPolicy: + """Create account federation policy. + + :param policy: :class:`FederationPolicy` (optional) + :param policy_id: str (optional) + The identifier for the federation policy. If unspecified, the id will be assigned by Databricks. + + :returns: :class:`FederationPolicy` + """ + body = policy.as_dict() + headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } + + res = self._api.do('POST', + f'/api/2.0/accounts/{self._api.account_id}/federationPolicies', + query=query, + body=body, + headers=headers) + return FederationPolicy.from_dict(res) + + def delete(self, policy_id: str): + """Delete account federation policy. + + :param policy_id: str + + + """ + + headers = {'Accept': 'application/json', } + + self._api.do('DELETE', + f'/api/2.0/accounts/{self._api.account_id}/federationPolicies/{policy_id}', + headers=headers) + + def get(self, policy_id: str) -> FederationPolicy: + """Get account federation policy. + + :param policy_id: str + + :returns: :class:`FederationPolicy` + """ + + headers = {'Accept': 'application/json', } + + res = self._api.do('GET', + f'/api/2.0/accounts/{self._api.account_id}/federationPolicies/{policy_id}', + headers=headers) + return FederationPolicy.from_dict(res) + + def list(self, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None) -> Iterator[FederationPolicy]: + """List account federation policies. + + :param page_size: int (optional) + :param page_token: str (optional) + + :returns: Iterator over :class:`FederationPolicy` + """ + + query = {} + if page_size is not None: query['page_size'] = page_size + if page_token is not None: query['page_token'] = page_token + headers = {'Accept': 'application/json', } + + while True: + json = self._api.do('GET', + f'/api/2.0/accounts/{self._api.account_id}/federationPolicies', + query=query, + headers=headers) + if 'policies' in json: + for v in json['policies']: + yield FederationPolicy.from_dict(v) + if 'next_page_token' not in json or not json['next_page_token']: + return + query['page_token'] = json['next_page_token'] + + def update(self, + policy_id: str, + update_mask: str, + *, + policy: Optional[FederationPolicy] = None) -> FederationPolicy: + """Update account federation policy. + + :param policy_id: str + :param update_mask: str + Field mask is required to be passed into the PATCH request. Field mask specifies which fields of the + setting payload will be updated. The field mask needs to be supplied as single string. To specify + multiple fields in the field mask, use comma as the separator (no space). + :param policy: :class:`FederationPolicy` (optional) + + :returns: :class:`FederationPolicy` + """ + body = policy.as_dict() + headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } + + res = self._api.do('PATCH', + f'/api/2.0/accounts/{self._api.account_id}/federationPolicies/{policy_id}', + query=query, + body=body, + headers=headers) + return FederationPolicy.from_dict(res) + + class CustomAppIntegrationAPI: """These APIs enable administrators to manage custom OAuth app integrations, which is required for adding/using Custom OAuth App Integration like Tableau Cloud for Databricks in AWS cloud.""" @@ -1086,6 +1377,176 @@ def update(self, integration_id: str, *, token_access_policy: Optional[TokenAcce headers=headers) +class ServicePrincipalFederationPolicyAPI: + """These APIs manage service principal federation policies. + + Service principal federation, also known as Workload Identity Federation, allows your automated workloads + running outside of Databricks to securely access Databricks APIs without the need for Databricks secrets. + With Workload Identity Federation, your application (or workload) authenticates to Databricks as a + Databricks service principal, using tokens provided by the workload runtime. + + Databricks strongly recommends using Workload Identity Federation to authenticate to Databricks from + automated workloads, over alternatives such as OAuth client secrets or Personal Access Tokens, whenever + possible. Workload Identity Federation is supported by many popular services, including Github Actions, + Azure DevOps, GitLab, Terraform Cloud, and Kubernetes clusters, among others. + + Workload identity federation is configured in your Databricks account using a service principal federation + policy. A service principal federation policy specifies: * which IdP, or issuer, the service principal is + allowed to authenticate from * which workload identity, or subject, is allowed to authenticate as the + Databricks service principal + + To configure a federation policy, you provide the following: * The required token __issuer__, as specified + in the “iss” claim of workload identity tokens. The issuer is an https URL that identifies the + workload identity provider. * The required token __subject__, as specified in the “sub” claim of + workload identity tokens. The subject uniquely identifies the workload in the workload runtime + environment. * The allowed token __audiences__, as specified in the “aud” claim of workload identity + tokens. The audience is intended to represent the recipient of the token. As long as the audience in the + token matches at least one audience in the policy, the token is considered a match. If unspecified, the + default value is your Databricks account id. * Optionally, the public keys used to validate the signature + of the workload identity tokens, in JWKS format. If unspecified (recommended), Databricks automatically + fetches the public keys from the issuer’s well known endpoint. Databricks strongly recommends relying on + the issuer’s well known endpoint for discovering public keys. + + An example service principal federation policy, for a Github Actions workload, is: ``` issuer: + "https://token.actions.githubusercontent.com" audiences: ["https://github.com/my-github-org"] subject: + "repo:my-github-org/my-repo:environment:prod" ``` + + An example JWT token body that matches this policy and could be used to authenticate to Databricks is: ``` + { "iss": "https://token.actions.githubusercontent.com", "aud": "https://github.com/my-github-org", "sub": + "repo:my-github-org/my-repo:environment:prod" } ``` + + You may also need to configure the workload runtime to generate tokens for your workloads. + + You do not need to configure an OAuth application in Databricks to use token federation.""" + + def __init__(self, api_client): + self._api = api_client + + def create(self, + service_principal_id: int, + *, + policy: Optional[FederationPolicy] = None, + policy_id: Optional[str] = None) -> FederationPolicy: + """Create service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy: :class:`FederationPolicy` (optional) + :param policy_id: str (optional) + The identifier for the federation policy. If unspecified, the id will be assigned by Databricks. + + :returns: :class:`FederationPolicy` + """ + body = policy.as_dict() + headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } + + res = self._api.do( + 'POST', + f'/api/2.0/accounts/{self._api.account_id}/servicePrincipals/{service_principal_id}/federationPolicies', + query=query, + body=body, + headers=headers) + return FederationPolicy.from_dict(res) + + def delete(self, service_principal_id: int, policy_id: str): + """Delete service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + + + """ + + headers = {'Accept': 'application/json', } + + self._api.do( + 'DELETE', + f'/api/2.0/accounts/{self._api.account_id}/servicePrincipals/{service_principal_id}/federationPolicies/{policy_id}', + headers=headers) + + def get(self, service_principal_id: int, policy_id: str) -> FederationPolicy: + """Get service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + + :returns: :class:`FederationPolicy` + """ + + headers = {'Accept': 'application/json', } + + res = self._api.do( + 'GET', + f'/api/2.0/accounts/{self._api.account_id}/servicePrincipals/{service_principal_id}/federationPolicies/{policy_id}', + headers=headers) + return FederationPolicy.from_dict(res) + + def list(self, + service_principal_id: int, + *, + page_size: Optional[int] = None, + page_token: Optional[str] = None) -> Iterator[FederationPolicy]: + """List service principal federation policies. + + :param service_principal_id: int + The service principal id for the federation policy. + :param page_size: int (optional) + :param page_token: str (optional) + + :returns: Iterator over :class:`FederationPolicy` + """ + + query = {} + if page_size is not None: query['page_size'] = page_size + if page_token is not None: query['page_token'] = page_token + headers = {'Accept': 'application/json', } + + while True: + json = self._api.do( + 'GET', + f'/api/2.0/accounts/{self._api.account_id}/servicePrincipals/{service_principal_id}/federationPolicies', + query=query, + headers=headers) + if 'policies' in json: + for v in json['policies']: + yield FederationPolicy.from_dict(v) + if 'next_page_token' not in json or not json['next_page_token']: + return + query['page_token'] = json['next_page_token'] + + def update(self, + service_principal_id: int, + policy_id: str, + update_mask: str, + *, + policy: Optional[FederationPolicy] = None) -> FederationPolicy: + """Update service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + :param update_mask: str + Field mask is required to be passed into the PATCH request. Field mask specifies which fields of the + setting payload will be updated. The field mask needs to be supplied as single string. To specify + multiple fields in the field mask, use comma as the separator (no space). + :param policy: :class:`FederationPolicy` (optional) + + :returns: :class:`FederationPolicy` + """ + body = policy.as_dict() + headers = {'Accept': 'application/json', 'Content-Type': 'application/json', } + + res = self._api.do( + 'PATCH', + f'/api/2.0/accounts/{self._api.account_id}/servicePrincipals/{service_principal_id}/federationPolicies/{policy_id}', + query=query, + body=body, + headers=headers) + return FederationPolicy.from_dict(res) + + class ServicePrincipalSecretsAPI: """These APIs enable administrators to manage service principal secrets. diff --git a/databricks/sdk/service/pipelines.py b/databricks/sdk/service/pipelines.py index 0ded4a83b..8f8b015c5 100755 --- a/databricks/sdk/service/pipelines.py +++ b/databricks/sdk/service/pipelines.py @@ -11,7 +11,7 @@ from typing import Callable, Dict, Iterator, List, Optional from ..errors import OperationFailed -from ._internal import Wait, _enum, _from_dict, _repeated_dict +from ._internal import Wait, _enum, _from_dict, _repeated_dict, _repeated_enum _LOG = logging.getLogger('databricks.sdk') @@ -2105,7 +2105,7 @@ class RestartWindow: """An integer between 0 and 23 denoting the start hour for the restart window in the 24-hour day. Continuous pipeline restart is triggered only within a five-hour window starting at this hour.""" - days_of_week: Optional[RestartWindowDaysOfWeek] = None + days_of_week: Optional[List[RestartWindowDaysOfWeek]] = None """Days of week in which the restart is allowed to happen (within a five-hour window starting at start_hour). If not specified all days of the week will be used.""" @@ -2117,7 +2117,7 @@ class RestartWindow: def as_dict(self) -> dict: """Serializes the RestartWindow into a dictionary suitable for use as a JSON request body.""" body = {} - if self.days_of_week is not None: body['days_of_week'] = self.days_of_week.value + if self.days_of_week: body['days_of_week'] = [v.value for v in self.days_of_week] if self.start_hour is not None: body['start_hour'] = self.start_hour if self.time_zone_id is not None: body['time_zone_id'] = self.time_zone_id return body @@ -2125,7 +2125,7 @@ def as_dict(self) -> dict: def as_shallow_dict(self) -> dict: """Serializes the RestartWindow into a shallow dictionary of its immediate attributes.""" body = {} - if self.days_of_week is not None: body['days_of_week'] = self.days_of_week + if self.days_of_week: body['days_of_week'] = self.days_of_week if self.start_hour is not None: body['start_hour'] = self.start_hour if self.time_zone_id is not None: body['time_zone_id'] = self.time_zone_id return body @@ -2133,7 +2133,7 @@ def as_shallow_dict(self) -> dict: @classmethod def from_dict(cls, d: Dict[str, any]) -> RestartWindow: """Deserializes the RestartWindow from a dictionary.""" - return cls(days_of_week=_enum(d, 'days_of_week', RestartWindowDaysOfWeek), + return cls(days_of_week=_repeated_enum(d, 'days_of_week', RestartWindowDaysOfWeek), start_hour=d.get('start_hour', None), time_zone_id=d.get('time_zone_id', None)) diff --git a/databricks/sdk/version.py b/databricks/sdk/version.py index 31a9ee722..eb9b6f12e 100644 --- a/databricks/sdk/version.py +++ b/databricks/sdk/version.py @@ -1 +1 @@ -__version__ = '0.39.0' +__version__ = '0.40.0' diff --git a/docs/account/oauth2/federation_policy.rst b/docs/account/oauth2/federation_policy.rst new file mode 100644 index 000000000..4bee8675d --- /dev/null +++ b/docs/account/oauth2/federation_policy.rst @@ -0,0 +1,99 @@ +``a.federation_policy``: Account Federation Policies +==================================================== +.. currentmodule:: databricks.sdk.service.oauth2 + +.. py:class:: AccountFederationPolicyAPI + + These APIs manage account federation policies. + + Account federation policies allow users and service principals in your Databricks account to securely + access Databricks APIs using tokens from your trusted identity providers (IdPs). + + With token federation, your users and service principals can exchange tokens from your IdP for Databricks + OAuth tokens, which can be used to access Databricks APIs. Token federation eliminates the need to manage + Databricks secrets, and allows you to centralize management of token issuance policies in your IdP. + Databricks token federation is typically used in combination with [SCIM], so users in your IdP are + synchronized into your Databricks account. + + Token federation is configured in your Databricks account using an account federation policy. An account + federation policy specifies: * which IdP, or issuer, your Databricks account should accept tokens from * + how to determine which Databricks user, or subject, a token is issued for + + To configure a federation policy, you provide the following: * The required token __issuer__, as specified + in the “iss” claim of your tokens. The issuer is an https URL that identifies your IdP. * The allowed + token __audiences__, as specified in the “aud” claim of your tokens. This identifier is intended to + represent the recipient of the token. As long as the audience in the token matches at least one audience + in the policy, the token is considered a match. If unspecified, the default value is your Databricks + account id. * The __subject claim__, which indicates which token claim contains the Databricks username of + the user the token was issued for. If unspecified, the default value is “sub”. * Optionally, the + public keys used to validate the signature of your tokens, in JWKS format. If unspecified (recommended), + Databricks automatically fetches the public keys from your issuer’s well known endpoint. Databricks + strongly recommends relying on your issuer’s well known endpoint for discovering public keys. + + An example federation policy is: ``` issuer: "https://idp.mycompany.com/oidc" audiences: ["databricks"] + subject_claim: "sub" ``` + + An example JWT token body that matches this policy and could be used to authenticate to Databricks as user + `username@mycompany.com` is: ``` { "iss": "https://idp.mycompany.com/oidc", "aud": "databricks", "sub": + "username@mycompany.com" } ``` + + You may also need to configure your IdP to generate tokens for your users to exchange with Databricks, if + your users do not already have the ability to generate tokens that are compatible with your federation + policy. + + You do not need to configure an OAuth application in Databricks to use token federation. + + [SCIM]: https://docs.databricks.com/admin/users-groups/scim/index.html + + .. py:method:: create( [, policy: Optional[FederationPolicy], policy_id: Optional[str]]) -> FederationPolicy + + Create account federation policy. + + :param policy: :class:`FederationPolicy` (optional) + :param policy_id: str (optional) + The identifier for the federation policy. If unspecified, the id will be assigned by Databricks. + + :returns: :class:`FederationPolicy` + + + .. py:method:: delete(policy_id: str) + + Delete account federation policy. + + :param policy_id: str + + + + + .. py:method:: get(policy_id: str) -> FederationPolicy + + Get account federation policy. + + :param policy_id: str + + :returns: :class:`FederationPolicy` + + + .. py:method:: list( [, page_size: Optional[int], page_token: Optional[str]]) -> Iterator[FederationPolicy] + + List account federation policies. + + :param page_size: int (optional) + :param page_token: str (optional) + + :returns: Iterator over :class:`FederationPolicy` + + + .. py:method:: update(policy_id: str, update_mask: str [, policy: Optional[FederationPolicy]]) -> FederationPolicy + + Update account federation policy. + + :param policy_id: str + :param update_mask: str + Field mask is required to be passed into the PATCH request. Field mask specifies which fields of the + setting payload will be updated. The field mask needs to be supplied as single string. To specify + multiple fields in the field mask, use comma as the separator (no space). + :param policy: :class:`FederationPolicy` (optional) + + :returns: :class:`FederationPolicy` + \ No newline at end of file diff --git a/docs/account/oauth2/index.rst b/docs/account/oauth2/index.rst index a4663ef6b..745a3e721 100644 --- a/docs/account/oauth2/index.rst +++ b/docs/account/oauth2/index.rst @@ -8,6 +8,8 @@ Configure OAuth 2.0 application registrations for Databricks :maxdepth: 1 custom_app_integration + federation_policy o_auth_published_apps published_app_integration + service_principal_federation_policy service_principal_secrets \ No newline at end of file diff --git a/docs/account/oauth2/service_principal_federation_policy.rst b/docs/account/oauth2/service_principal_federation_policy.rst new file mode 100644 index 000000000..e4293c5f2 --- /dev/null +++ b/docs/account/oauth2/service_principal_federation_policy.rst @@ -0,0 +1,109 @@ +``a.service_principal_federation_policy``: Service Principal Federation Policies +================================================================================ +.. currentmodule:: databricks.sdk.service.oauth2 + +.. py:class:: ServicePrincipalFederationPolicyAPI + + These APIs manage service principal federation policies. + + Service principal federation, also known as Workload Identity Federation, allows your automated workloads + running outside of Databricks to securely access Databricks APIs without the need for Databricks secrets. + With Workload Identity Federation, your application (or workload) authenticates to Databricks as a + Databricks service principal, using tokens provided by the workload runtime. + + Databricks strongly recommends using Workload Identity Federation to authenticate to Databricks from + automated workloads, over alternatives such as OAuth client secrets or Personal Access Tokens, whenever + possible. Workload Identity Federation is supported by many popular services, including Github Actions, + Azure DevOps, GitLab, Terraform Cloud, and Kubernetes clusters, among others. + + Workload identity federation is configured in your Databricks account using a service principal federation + policy. A service principal federation policy specifies: * which IdP, or issuer, the service principal is + allowed to authenticate from * which workload identity, or subject, is allowed to authenticate as the + Databricks service principal + + To configure a federation policy, you provide the following: * The required token __issuer__, as specified + in the “iss” claim of workload identity tokens. The issuer is an https URL that identifies the + workload identity provider. * The required token __subject__, as specified in the “sub” claim of + workload identity tokens. The subject uniquely identifies the workload in the workload runtime + environment. * The allowed token __audiences__, as specified in the “aud” claim of workload identity + tokens. The audience is intended to represent the recipient of the token. As long as the audience in the + token matches at least one audience in the policy, the token is considered a match. If unspecified, the + default value is your Databricks account id. * Optionally, the public keys used to validate the signature + of the workload identity tokens, in JWKS format. If unspecified (recommended), Databricks automatically + fetches the public keys from the issuer’s well known endpoint. Databricks strongly recommends relying on + the issuer’s well known endpoint for discovering public keys. + + An example service principal federation policy, for a Github Actions workload, is: ``` issuer: + "https://token.actions.githubusercontent.com" audiences: ["https://github.com/my-github-org"] subject: + "repo:my-github-org/my-repo:environment:prod" ``` + + An example JWT token body that matches this policy and could be used to authenticate to Databricks is: ``` + { "iss": "https://token.actions.githubusercontent.com", "aud": "https://github.com/my-github-org", "sub": + "repo:my-github-org/my-repo:environment:prod" } ``` + + You may also need to configure the workload runtime to generate tokens for your workloads. + + You do not need to configure an OAuth application in Databricks to use token federation. + + .. py:method:: create(service_principal_id: int [, policy: Optional[FederationPolicy], policy_id: Optional[str]]) -> FederationPolicy + + Create service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy: :class:`FederationPolicy` (optional) + :param policy_id: str (optional) + The identifier for the federation policy. If unspecified, the id will be assigned by Databricks. + + :returns: :class:`FederationPolicy` + + + .. py:method:: delete(service_principal_id: int, policy_id: str) + + Delete service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + + + + + .. py:method:: get(service_principal_id: int, policy_id: str) -> FederationPolicy + + Get service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + + :returns: :class:`FederationPolicy` + + + .. py:method:: list(service_principal_id: int [, page_size: Optional[int], page_token: Optional[str]]) -> Iterator[FederationPolicy] + + List service principal federation policies. + + :param service_principal_id: int + The service principal id for the federation policy. + :param page_size: int (optional) + :param page_token: str (optional) + + :returns: Iterator over :class:`FederationPolicy` + + + .. py:method:: update(service_principal_id: int, policy_id: str, update_mask: str [, policy: Optional[FederationPolicy]]) -> FederationPolicy + + Update service principal federation policy. + + :param service_principal_id: int + The service principal id for the federation policy. + :param policy_id: str + :param update_mask: str + Field mask is required to be passed into the PATCH request. Field mask specifies which fields of the + setting payload will be updated. The field mask needs to be supplied as single string. To specify + multiple fields in the field mask, use comma as the separator (no space). + :param policy: :class:`FederationPolicy` (optional) + + :returns: :class:`FederationPolicy` + \ No newline at end of file diff --git a/docs/dbdataclasses/catalog.rst b/docs/dbdataclasses/catalog.rst index 5b5fbb379..84f3c9867 100644 --- a/docs/dbdataclasses/catalog.rst +++ b/docs/dbdataclasses/catalog.rst @@ -745,12 +745,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: CATALOG :value: "CATALOG" + .. py:attribute:: CREDENTIAL + :value: "CREDENTIAL" + .. py:attribute:: EXTERNAL_LOCATION :value: "EXTERNAL_LOCATION" - .. py:attribute:: SERVICE_CREDENTIAL - :value: "SERVICE_CREDENTIAL" - .. py:attribute:: STORAGE_CREDENTIAL :value: "STORAGE_CREDENTIAL" @@ -1460,12 +1460,12 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: CATALOG :value: "CATALOG" + .. py:attribute:: CREDENTIAL + :value: "CREDENTIAL" + .. py:attribute:: EXTERNAL_LOCATION :value: "EXTERNAL_LOCATION" - .. py:attribute:: SERVICE_CREDENTIAL - :value: "SERVICE_CREDENTIAL" - .. py:attribute:: STORAGE_CREDENTIAL :value: "STORAGE_CREDENTIAL" diff --git a/docs/dbdataclasses/compute.rst b/docs/dbdataclasses/compute.rst index 0066f0374..9c628c476 100644 --- a/docs/dbdataclasses/compute.rst +++ b/docs/dbdataclasses/compute.rst @@ -316,10 +316,20 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: DataSecurityMode Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will choose the most appropriate access mode depending on your compute configuration. * `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias for `SINGLE_USER`. + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. Most programming languages, cluster features and data governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data and credentials. Most data governance features are supported in this mode. But programming languages and cluster features might be limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: * `LEGACY_TABLE_ACL`: This mode is for users migrating from legacy Table ACL clusters. * `LEGACY_PASSTHROUGH`: This mode is for users migrating from legacy Passthrough on high concurrency clusters. * `LEGACY_SINGLE_USER`: This mode is for users migrating from legacy Passthrough on standard clusters. * `LEGACY_SINGLE_USER_STANDARD`: This mode provides a way that doesn’t have UC nor passthrough enabled. + .. py:attribute:: DATA_SECURITY_MODE_AUTO + :value: "DATA_SECURITY_MODE_AUTO" + + .. py:attribute:: DATA_SECURITY_MODE_DEDICATED + :value: "DATA_SECURITY_MODE_DEDICATED" + + .. py:attribute:: DATA_SECURITY_MODE_STANDARD + :value: "DATA_SECURITY_MODE_STANDARD" + .. py:attribute:: LEGACY_PASSTHROUGH :value: "LEGACY_PASSTHROUGH" @@ -782,6 +792,15 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. py:class:: Kind + + The kind of compute described by this compute specification. + Depending on `kind`, different validations and default values will be applied. + The first usage of this value is for the simple cluster form where it sets `kind = CLASSIC_PREVIEW`. + + .. py:attribute:: CLASSIC_PREVIEW + :value: "CLASSIC_PREVIEW" + .. py:class:: Language .. py:attribute:: PYTHON diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index 374c48351..cbb4059a1 100644 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -103,6 +103,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: CleanRoomsNotebookTask + :members: + :undoc-members: + .. autoclass:: ClusterInstance :members: :undoc-members: @@ -392,7 +396,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: JobsHealthMetric Specifies the health metric that is being evaluated for a particular health rule. - * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Private Preview. + * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`: An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_SECONDS`: An estimate of the maximum consumer delay across all streams. This metric is in Public Preview. * `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all streams. This metric is in Public Preview. .. py:attribute:: RUN_DURATION_SECONDS :value: "RUN_DURATION_SECONDS" @@ -975,7 +979,7 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: TriggerType The type of trigger that fired this run. - * `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler. * `ONE_TIME`: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is triggered by a table update. + * `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler. * `ONE_TIME`: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. .. py:attribute:: FILE_ARRIVAL :value: "FILE_ARRIVAL" diff --git a/docs/dbdataclasses/oauth2.rst b/docs/dbdataclasses/oauth2.rst index 6265f6648..70e09ab05 100644 --- a/docs/dbdataclasses/oauth2.rst +++ b/docs/dbdataclasses/oauth2.rst @@ -40,6 +40,10 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: FederationPolicy + :members: + :undoc-members: + .. autoclass:: GetCustomAppIntegrationOutput :members: :undoc-members: @@ -60,10 +64,18 @@ These dataclasses are used in the SDK to represent API requests and responses fo :members: :undoc-members: +.. autoclass:: ListFederationPoliciesResponse + :members: + :undoc-members: + .. autoclass:: ListServicePrincipalSecretsResponse :members: :undoc-members: +.. autoclass:: OidcFederationPolicy + :members: + :undoc-members: + .. autoclass:: PublishedAppOutput :members: :undoc-members: diff --git a/docs/workspace/compute/clusters.rst b/docs/workspace/compute/clusters.rst index 24fe2d253..c51a61602 100644 --- a/docs/workspace/compute/clusters.rst +++ b/docs/workspace/compute/clusters.rst @@ -71,7 +71,7 @@ - .. py:method:: create(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] + .. py:method:: create(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] Usage: @@ -152,13 +152,19 @@ :param data_security_mode: :class:`DataSecurityMode` (optional) Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are - not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a - single user specified in `single_user_name`. Most programming languages, cluster features and data - governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be - shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data - and credentials. Most data governance features are supported in this mode. But programming languages - and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias + for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple + users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: + A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. + Most programming languages, cluster features and data governance features are available in this + mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are + fully isolated so that they cannot see each other's data and credentials. Most data governance + features are supported in this mode. But programming languages and cluster features might be + limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -190,6 +196,17 @@ logs are sent to `//init_scripts`. :param instance_pool_id: str (optional) The optional ID of the instance pool to which the cluster belongs. + :param is_single_node: bool (optional) + This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, + and `num_workers` + :param kind: :class:`Kind` (optional) + The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = CLASSIC_PREVIEW`. :param node_type_id: str (optional) This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute @@ -236,6 +253,11 @@ SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param use_ml_runtime: bool (optional) + This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not. :param workload_type: :class:`WorkloadType` (optional) :returns: @@ -243,7 +265,7 @@ See :method:wait_get_cluster_running for more details. - .. py:method:: create_and_wait(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails + .. py:method:: create_and_wait(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails .. py:method:: delete(cluster_id: str) -> Wait[ClusterDetails] @@ -292,7 +314,7 @@ .. py:method:: delete_and_wait(cluster_id: str, timeout: datetime.timedelta = 0:20:00) -> ClusterDetails - .. py:method:: edit(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] + .. py:method:: edit(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] Usage: @@ -380,13 +402,19 @@ :param data_security_mode: :class:`DataSecurityMode` (optional) Data security mode decides what data governance model to use when accessing data from a cluster. - * `NONE`: No security isolation for multiple users sharing the cluster. Data governance features are - not available in this mode. * `SINGLE_USER`: A secure cluster that can only be exclusively used by a - single user specified in `single_user_name`. Most programming languages, cluster features and data - governance features are available in this mode. * `USER_ISOLATION`: A secure cluster that can be - shared by multiple users. Cluster users are fully isolated so that they cannot see each other's data - and credentials. Most data governance features are supported in this mode. But programming languages - and cluster features might be limited. + The following modes can only be used with `kind`. * `DATA_SECURITY_MODE_AUTO`: Databricks will + choose the most appropriate access mode depending on your compute configuration. * + `DATA_SECURITY_MODE_STANDARD`: Alias for `USER_ISOLATION`. * `DATA_SECURITY_MODE_DEDICATED`: Alias + for `SINGLE_USER`. + + The following modes can be used regardless of `kind`. * `NONE`: No security isolation for multiple + users sharing the cluster. Data governance features are not available in this mode. * `SINGLE_USER`: + A secure cluster that can only be exclusively used by a single user specified in `single_user_name`. + Most programming languages, cluster features and data governance features are available in this + mode. * `USER_ISOLATION`: A secure cluster that can be shared by multiple users. Cluster users are + fully isolated so that they cannot see each other's data and credentials. Most data governance + features are supported in this mode. But programming languages and cluster features might be + limited. The following modes are deprecated starting with Databricks Runtime 15.0 and will be removed for future Databricks Runtime versions: @@ -418,6 +446,17 @@ logs are sent to `//init_scripts`. :param instance_pool_id: str (optional) The optional ID of the instance pool to which the cluster belongs. + :param is_single_node: bool (optional) + This field can only be used with `kind`. + + When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, + and `num_workers` + :param kind: :class:`Kind` (optional) + The kind of compute described by this compute specification. + + Depending on `kind`, different validations and default values will be applied. + + The first usage of this value is for the simple cluster form where it sets `kind = CLASSIC_PREVIEW`. :param node_type_id: str (optional) This field encodes, through a single value, the resources available to each of the Spark nodes in this cluster. For example, the Spark nodes can be provisioned and optimized for memory or compute @@ -464,6 +503,11 @@ SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param use_ml_runtime: bool (optional) + This field can only be used with `kind`. + + `effective_spark_version` is determined by `spark_version` (DBR release), this field + `use_ml_runtime`, and whether `node_type_id` is gpu node or not. :param workload_type: :class:`WorkloadType` (optional) :returns: @@ -471,7 +515,7 @@ See :method:wait_get_cluster_running for more details. - .. py:method:: edit_and_wait(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails + .. py:method:: edit_and_wait(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails .. py:method:: ensure_cluster_is_running(cluster_id: str) diff --git a/docs/workspace/dashboards/lakeview.rst b/docs/workspace/dashboards/lakeview.rst index b8dceeb9e..c37479dcb 100644 --- a/docs/workspace/dashboards/lakeview.rst +++ b/docs/workspace/dashboards/lakeview.rst @@ -174,7 +174,7 @@ :returns: Iterator over :class:`Subscription` - .. py:method:: migrate(source_dashboard_id: str [, display_name: Optional[str], parent_path: Optional[str]]) -> Dashboard + .. py:method:: migrate(source_dashboard_id: str [, display_name: Optional[str], parent_path: Optional[str], update_parameter_syntax: Optional[bool]]) -> Dashboard Migrate dashboard. @@ -186,6 +186,9 @@ Display name for the new Lakeview dashboard. :param parent_path: str (optional) The workspace path of the folder to contain the migrated Lakeview dashboard. + :param update_parameter_syntax: bool (optional) + Flag to indicate if mustache parameter syntax ({{ param }}) should be auto-updated to named syntax + (:param) when converting datasets in the dashboard. :returns: :class:`Dashboard`