diff --git a/.codegen/_openapi_sha b/.codegen/_openapi_sha index 3fcbd79dd..3fbd74142 100644 --- a/.codegen/_openapi_sha +++ b/.codegen/_openapi_sha @@ -1 +1 @@ -95e08d8bc631ac93d68e3846aea3cb04a2913495 \ No newline at end of file +033bcb9242b006001e2cf3956896711681de1a8c \ No newline at end of file diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 277d098b5..9bb0298b3 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,3 +11,15 @@ ### Internal Changes ### API Changes +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.ClusterAttributes`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.ClusterDetails`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.ClusterSpec`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.CreateCluster`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.CreateInstancePool`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.EditCluster`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.EditInstancePool`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.GetInstancePool`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.InstancePoolAndStats`. +* Added `remote_disk_throughput` and `total_initial_remote_disk_size` fields for `databricks.sdk.service.compute.UpdateClusterResource`. +* Added `r` enum value for `databricks.sdk.service.compute.Language`. +* Added `continuous` and `continuous_restart` enum values for `databricks.sdk.service.jobs.TriggerType`. diff --git a/databricks/sdk/__init__.py b/databricks/sdk/__init__.py index 5e0fb4c62..3f144d309 100755 --- a/databricks/sdk/__init__.py +++ b/databricks/sdk/__init__.py @@ -1038,7 +1038,7 @@ def ip_access_lists(self) -> pkg_settings.AccountIpAccessListsAPI: @property def log_delivery(self) -> pkg_billing.LogDeliveryAPI: - """These APIs manage Log delivery configurations for this account.""" + """These APIs manage log delivery configurations for this account.""" return self._log_delivery @property diff --git a/databricks/sdk/service/billing.py b/databricks/sdk/service/billing.py index 763cc0501..8c61275b5 100755 --- a/databricks/sdk/service/billing.py +++ b/databricks/sdk/service/billing.py @@ -2022,8 +2022,57 @@ def update(self, budget_id: str, budget: UpdateBudgetConfigurationBudget) -> Upd class LogDeliveryAPI: - """These APIs manage Log delivery configurations for this account. Log delivery configs enable you to - configure the delivery of the specified type of logs to your storage account.""" + """These APIs manage log delivery configurations for this account. The two supported log types for this API + are _billable usage logs_ and _audit logs_. This feature is in Public Preview. This feature works with all + account ID types. + + Log delivery works with all account types. However, if your account is on the E2 version of the platform + or on a select custom plan that allows multiple workspaces per account, you can optionally configure + different storage destinations for each workspace. Log delivery status is also provided to know the latest + status of log delivery attempts. + + The high-level flow of billable usage delivery: + + 1. **Create storage**: In AWS, [create a new AWS S3 bucket] with a specific bucket policy. Using + Databricks APIs, call the Account API to create a [storage configuration object](:method:Storage/Create) + that uses the bucket name. + + 2. **Create credentials**: In AWS, create the appropriate AWS IAM role. For full details, including the + required IAM role policies and trust relationship, see [Billable usage log delivery]. Using Databricks + APIs, call the Account API to create a [credential configuration object](:method:Credentials/Create) that + uses the IAM role's ARN. + + 3. **Create log delivery configuration**: Using Databricks APIs, call the Account API to [create a log + delivery configuration](:method:LogDelivery/Create) that uses the credential and storage configuration + objects from previous steps. You can specify if the logs should include all events of that log type in + your account (_Account level_ delivery) or only events for a specific set of workspaces (_workspace level_ + delivery). Account level log delivery applies to all current and future workspaces plus account level + logs, while workspace level log delivery solely delivers logs related to the specified workspaces. You can + create multiple types of delivery configurations per account. + + For billable usage delivery: * For more information about billable usage logs, see [Billable usage log + delivery]. For the CSV schema, see the [Usage page]. * The delivery location is + `//billable-usage/csv/`, where `` is the name of the optional delivery path + prefix you set up during log delivery configuration. Files are named + `workspaceId=-usageMonth=.csv`. * All billable usage logs apply to specific + workspaces (_workspace level_ logs). You can aggregate usage for your entire account by creating an + _account level_ delivery configuration that delivers logs for all current and future workspaces in your + account. * The files are delivered daily by overwriting the month's CSV file for each workspace. + + For audit log delivery: * For more information about about audit log delivery, see [Audit log delivery], + which includes information about the used JSON schema. * The delivery location is + `//workspaceId=/date=/auditlogs_.json`. + Files may get overwritten with the same content multiple times to achieve exactly-once delivery. * If the + audit log delivery configuration included specific workspace IDs, only _workspace-level_ audit logs for + those workspaces are delivered. If the log delivery configuration applies to the entire account (_account + level_ delivery configuration), the audit log delivery includes workspace-level audit logs for all + workspaces in the account as well as account-level audit logs. See [Audit log delivery] for details. * + Auditable events are typically available in logs within 15 minutes. + + [Audit log delivery]: https://docs.databricks.com/administration-guide/account-settings/audit-logs.html + [Billable usage log delivery]: https://docs.databricks.com/administration-guide/account-settings/billable-usage-delivery.html + [Usage page]: https://docs.databricks.com/administration-guide/account-settings/usage.html + [create a new AWS S3 bucket]: https://docs.databricks.com/administration-guide/account-api/aws-storage.html""" def __init__(self, api_client): self._api = api_client diff --git a/databricks/sdk/service/catalog.py b/databricks/sdk/service/catalog.py index 3cd01d05b..102bae13f 100755 --- a/databricks/sdk/service/catalog.py +++ b/databricks/sdk/service/catalog.py @@ -10655,6 +10655,7 @@ def get(self, metastore_id: str) -> AccountsMetastoreInfo: def list(self) -> Iterator[MetastoreInfo]: """Gets all Unity Catalog metastores associated with an account specified by ID. + :returns: Iterator over :class:`MetastoreInfo` """ @@ -12302,6 +12303,7 @@ def create(self, name: str, *, region: Optional[str] = None, storage_root: Optio def current(self) -> MetastoreAssignment: """Gets the metastore assignment for the workspace being accessed. + :returns: :class:`MetastoreAssignment` """ @@ -12389,6 +12391,7 @@ def summary(self) -> GetMetastoreSummaryResponse: """Gets information about a metastore. This summary includes the storage credential, the cloud vendor, the cloud region, and the global metastore ID. + :returns: :class:`GetMetastoreSummaryResponse` """ diff --git a/databricks/sdk/service/compute.py b/databricks/sdk/service/compute.py index e04a09fca..5092314a0 100755 --- a/databricks/sdk/service/compute.py +++ b/databricks/sdk/service/compute.py @@ -832,6 +832,10 @@ class ClusterAttributes: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -867,6 +871,10 @@ class ClusterAttributes: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -917,6 +925,8 @@ def as_dict(self) -> dict: body["node_type_id"] = self.node_type_id if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -929,6 +939,8 @@ def as_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -976,6 +988,8 @@ def as_shallow_dict(self) -> dict: body["node_type_id"] = self.node_type_id if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -988,6 +1002,8 @@ def as_shallow_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -1017,12 +1033,14 @@ def from_dict(cls, d: Dict[str, Any]) -> ClusterAttributes: kind=_enum(d, "kind", Kind), node_type_id=d.get("node_type_id", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), spark_env_vars=d.get("spark_env_vars", None), spark_version=d.get("spark_version", None), ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -1270,6 +1288,10 @@ class ClusterDetails: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -1336,6 +1358,10 @@ class ClusterDetails: """Information about why the cluster was terminated. This field only appears when the cluster is in a `TERMINATING` or `TERMINATED` state.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -1414,6 +1440,8 @@ def as_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -1440,6 +1468,8 @@ def as_dict(self) -> dict: body["terminated_time"] = self.terminated_time if self.termination_reason: body["termination_reason"] = self.termination_reason.as_dict() + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -1515,6 +1545,8 @@ def as_shallow_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -1541,6 +1573,8 @@ def as_shallow_dict(self) -> dict: body["terminated_time"] = self.terminated_time if self.termination_reason: body["termination_reason"] = self.termination_reason + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -1584,6 +1618,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ClusterDetails: node_type_id=d.get("node_type_id", None), num_workers=d.get("num_workers", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), @@ -1597,6 +1632,7 @@ def from_dict(cls, d: Dict[str, Any]) -> ClusterDetails: state_message=d.get("state_message", None), terminated_time=d.get("terminated_time", None), termination_reason=_from_dict(d, "termination_reason", TerminationReason), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -2415,6 +2451,10 @@ class ClusterSpec: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -2454,6 +2494,10 @@ class ClusterSpec: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -2510,6 +2554,8 @@ def as_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -2522,6 +2568,8 @@ def as_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -2575,6 +2623,8 @@ def as_shallow_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -2587,6 +2637,8 @@ def as_shallow_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -2619,12 +2671,14 @@ def from_dict(cls, d: Dict[str, Any]) -> ClusterSpec: node_type_id=d.get("node_type_id", None), num_workers=d.get("num_workers", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), spark_env_vars=d.get("spark_env_vars", None), spark_version=d.get("spark_version", None), ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -2919,6 +2973,10 @@ class CreateCluster: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -2954,6 +3012,10 @@ class CreateCluster: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -3012,6 +3074,8 @@ def as_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -3024,6 +3088,8 @@ def as_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -3079,6 +3145,8 @@ def as_shallow_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -3091,6 +3159,8 @@ def as_shallow_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -3124,12 +3194,14 @@ def from_dict(cls, d: Dict[str, Any]) -> CreateCluster: node_type_id=d.get("node_type_id", None), num_workers=d.get("num_workers", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), spark_env_vars=d.get("spark_env_vars", None), spark_version=d.get("spark_version", None), ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -3252,6 +3324,14 @@ class CreateInstancePool: started with the preloaded Spark version will start faster. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + def as_dict(self) -> dict: """Serializes the CreateInstancePool into a dictionary suitable for use as a JSON request body.""" body = {} @@ -3281,6 +3361,10 @@ def as_dict(self) -> dict: body["preloaded_docker_images"] = [v.as_dict() for v in self.preloaded_docker_images] if self.preloaded_spark_versions: body["preloaded_spark_versions"] = [v for v in self.preloaded_spark_versions] + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body def as_shallow_dict(self) -> dict: @@ -3312,6 +3396,10 @@ def as_shallow_dict(self) -> dict: body["preloaded_docker_images"] = self.preloaded_docker_images if self.preloaded_spark_versions: body["preloaded_spark_versions"] = self.preloaded_spark_versions + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body @classmethod @@ -3331,6 +3419,8 @@ def from_dict(cls, d: Dict[str, Any]) -> CreateInstancePool: node_type_id=d.get("node_type_id", None), preloaded_docker_images=_repeated_dict(d, "preloaded_docker_images", DockerImage), preloaded_spark_versions=d.get("preloaded_spark_versions", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), ) @@ -4228,6 +4318,10 @@ class EditCluster: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -4263,6 +4357,10 @@ class EditCluster: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -4321,6 +4419,8 @@ def as_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -4333,6 +4433,8 @@ def as_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -4388,6 +4490,8 @@ def as_shallow_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -4400,6 +4504,8 @@ def as_shallow_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -4433,12 +4539,14 @@ def from_dict(cls, d: Dict[str, Any]) -> EditCluster: node_type_id=d.get("node_type_id", None), num_workers=d.get("num_workers", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), spark_env_vars=d.get("spark_env_vars", None), spark_version=d.get("spark_version", None), ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -4498,6 +4606,14 @@ class EditInstancePool: min_idle_instances: Optional[int] = None """Minimum number of idle instances to keep in the instance pool""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + def as_dict(self) -> dict: """Serializes the EditInstancePool into a dictionary suitable for use as a JSON request body.""" body = {} @@ -4515,6 +4631,10 @@ def as_dict(self) -> dict: body["min_idle_instances"] = self.min_idle_instances if self.node_type_id is not None: body["node_type_id"] = self.node_type_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body def as_shallow_dict(self) -> dict: @@ -4534,6 +4654,10 @@ def as_shallow_dict(self) -> dict: body["min_idle_instances"] = self.min_idle_instances if self.node_type_id is not None: body["node_type_id"] = self.node_type_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body @classmethod @@ -4547,6 +4671,8 @@ def from_dict(cls, d: Dict[str, Any]) -> EditInstancePool: max_capacity=d.get("max_capacity", None), min_idle_instances=d.get("min_idle_instances", None), node_type_id=d.get("node_type_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), ) @@ -5506,6 +5632,10 @@ class GetInstancePool: started with the preloaded Spark version will start faster. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + state: Optional[InstancePoolState] = None """Current state of the instance pool.""" @@ -5515,6 +5645,10 @@ class GetInstancePool: status: Optional[InstancePoolStatus] = None """Status of failed pending instances in the pool.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + def as_dict(self) -> dict: """Serializes the GetInstancePool into a dictionary suitable for use as a JSON request body.""" body = {} @@ -5548,12 +5682,16 @@ def as_dict(self) -> dict: body["preloaded_docker_images"] = [v.as_dict() for v in self.preloaded_docker_images] if self.preloaded_spark_versions: body["preloaded_spark_versions"] = [v for v in self.preloaded_spark_versions] + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.state is not None: body["state"] = self.state.value if self.stats: body["stats"] = self.stats.as_dict() if self.status: body["status"] = self.status.as_dict() + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body def as_shallow_dict(self) -> dict: @@ -5589,12 +5727,16 @@ def as_shallow_dict(self) -> dict: body["preloaded_docker_images"] = self.preloaded_docker_images if self.preloaded_spark_versions: body["preloaded_spark_versions"] = self.preloaded_spark_versions + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.state is not None: body["state"] = self.state if self.stats: body["stats"] = self.stats if self.status: body["status"] = self.status + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body @classmethod @@ -5616,9 +5758,11 @@ def from_dict(cls, d: Dict[str, Any]) -> GetInstancePool: node_type_id=d.get("node_type_id", None), preloaded_docker_images=_repeated_dict(d, "preloaded_docker_images", DockerImage), preloaded_spark_versions=d.get("preloaded_spark_versions", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), state=_enum(d, "state", InstancePoolState), stats=_from_dict(d, "stats", InstancePoolStats), status=_from_dict(d, "status", InstancePoolStatus), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), ) @@ -6461,6 +6605,10 @@ class InstancePoolAndStats: started with the preloaded Spark version will start faster. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + state: Optional[InstancePoolState] = None """Current state of the instance pool.""" @@ -6470,6 +6618,10 @@ class InstancePoolAndStats: status: Optional[InstancePoolStatus] = None """Status of failed pending instances in the pool.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types.""" + def as_dict(self) -> dict: """Serializes the InstancePoolAndStats into a dictionary suitable for use as a JSON request body.""" body = {} @@ -6503,12 +6655,16 @@ def as_dict(self) -> dict: body["preloaded_docker_images"] = [v.as_dict() for v in self.preloaded_docker_images] if self.preloaded_spark_versions: body["preloaded_spark_versions"] = [v for v in self.preloaded_spark_versions] + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.state is not None: body["state"] = self.state.value if self.stats: body["stats"] = self.stats.as_dict() if self.status: body["status"] = self.status.as_dict() + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body def as_shallow_dict(self) -> dict: @@ -6544,12 +6700,16 @@ def as_shallow_dict(self) -> dict: body["preloaded_docker_images"] = self.preloaded_docker_images if self.preloaded_spark_versions: body["preloaded_spark_versions"] = self.preloaded_spark_versions + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.state is not None: body["state"] = self.state if self.stats: body["stats"] = self.stats if self.status: body["status"] = self.status + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size return body @classmethod @@ -6571,9 +6731,11 @@ def from_dict(cls, d: Dict[str, Any]) -> InstancePoolAndStats: node_type_id=d.get("node_type_id", None), preloaded_docker_images=_repeated_dict(d, "preloaded_docker_images", DockerImage), preloaded_spark_versions=d.get("preloaded_spark_versions", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), state=_enum(d, "state", InstancePoolState), stats=_from_dict(d, "stats", InstancePoolStats), status=_from_dict(d, "status", InstancePoolStatus), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), ) @@ -7066,6 +7228,7 @@ class Kind(Enum): class Language(Enum): PYTHON = "python" + R = "r" SCALA = "scala" SQL = "sql" @@ -9473,6 +9636,10 @@ class UpdateClusterResource: policy_id: Optional[str] = None """The ID of the cluster policy used to create the cluster if applicable.""" + remote_disk_throughput: Optional[int] = None + """If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + runtime_engine: Optional[RuntimeEngine] = None """Determines the cluster's runtime engine, either standard or Photon. @@ -9512,6 +9679,10 @@ class UpdateClusterResource: private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified.""" + total_initial_remote_disk_size: Optional[int] = None + """If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks.""" + use_ml_runtime: Optional[bool] = None """This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -9566,6 +9737,8 @@ def as_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine.value if self.single_user_name is not None: @@ -9578,6 +9751,8 @@ def as_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = [v for v in self.ssh_public_keys] + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -9629,6 +9804,8 @@ def as_shallow_dict(self) -> dict: body["num_workers"] = self.num_workers if self.policy_id is not None: body["policy_id"] = self.policy_id + if self.remote_disk_throughput is not None: + body["remote_disk_throughput"] = self.remote_disk_throughput if self.runtime_engine is not None: body["runtime_engine"] = self.runtime_engine if self.single_user_name is not None: @@ -9641,6 +9818,8 @@ def as_shallow_dict(self) -> dict: body["spark_version"] = self.spark_version if self.ssh_public_keys: body["ssh_public_keys"] = self.ssh_public_keys + if self.total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = self.total_initial_remote_disk_size if self.use_ml_runtime is not None: body["use_ml_runtime"] = self.use_ml_runtime if self.workload_type: @@ -9672,12 +9851,14 @@ def from_dict(cls, d: Dict[str, Any]) -> UpdateClusterResource: node_type_id=d.get("node_type_id", None), num_workers=d.get("num_workers", None), policy_id=d.get("policy_id", None), + remote_disk_throughput=d.get("remote_disk_throughput", None), runtime_engine=_enum(d, "runtime_engine", RuntimeEngine), single_user_name=d.get("single_user_name", None), spark_conf=d.get("spark_conf", None), spark_env_vars=d.get("spark_env_vars", None), spark_version=d.get("spark_version", None), ssh_public_keys=d.get("ssh_public_keys", None), + total_initial_remote_disk_size=d.get("total_initial_remote_disk_size", None), use_ml_runtime=d.get("use_ml_runtime", None), workload_type=_from_dict(d, "workload_type", WorkloadType), ) @@ -10257,11 +10438,13 @@ def create( node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, + remote_disk_throughput: Optional[int] = None, runtime_engine: Optional[RuntimeEngine] = None, single_user_name: Optional[str] = None, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + total_initial_remote_disk_size: Optional[int] = None, use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, ) -> Wait[ClusterDetails]: @@ -10405,6 +10588,9 @@ def create( provisioned. :param policy_id: str (optional) The ID of the cluster policy used to create the cluster if applicable. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED disks. :param runtime_engine: :class:`RuntimeEngine` (optional) Determines the cluster's runtime engine, either standard or Photon. @@ -10435,6 +10621,9 @@ def create( SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks. :param use_ml_runtime: bool (optional) This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -10494,6 +10683,8 @@ def create( body["num_workers"] = num_workers if policy_id is not None: body["policy_id"] = policy_id + if remote_disk_throughput is not None: + body["remote_disk_throughput"] = remote_disk_throughput if runtime_engine is not None: body["runtime_engine"] = runtime_engine.value if single_user_name is not None: @@ -10506,6 +10697,8 @@ def create( body["spark_version"] = spark_version if ssh_public_keys is not None: body["ssh_public_keys"] = [v for v in ssh_public_keys] + if total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = total_initial_remote_disk_size if use_ml_runtime is not None: body["use_ml_runtime"] = use_ml_runtime if workload_type is not None: @@ -10549,11 +10742,13 @@ def create_and_wait( node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, + remote_disk_throughput: Optional[int] = None, runtime_engine: Optional[RuntimeEngine] = None, single_user_name: Optional[str] = None, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + total_initial_remote_disk_size: Optional[int] = None, use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, timeout=timedelta(minutes=20), @@ -10582,12 +10777,14 @@ def create_and_wait( node_type_id=node_type_id, num_workers=num_workers, policy_id=policy_id, + remote_disk_throughput=remote_disk_throughput, runtime_engine=runtime_engine, single_user_name=single_user_name, spark_conf=spark_conf, spark_env_vars=spark_env_vars, spark_version=spark_version, ssh_public_keys=ssh_public_keys, + total_initial_remote_disk_size=total_initial_remote_disk_size, use_ml_runtime=use_ml_runtime, workload_type=workload_type, ).result(timeout=timeout) @@ -10649,11 +10846,13 @@ def edit( node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, + remote_disk_throughput: Optional[int] = None, runtime_engine: Optional[RuntimeEngine] = None, single_user_name: Optional[str] = None, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + total_initial_remote_disk_size: Optional[int] = None, use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, ) -> Wait[ClusterDetails]: @@ -10794,6 +10993,9 @@ def edit( provisioned. :param policy_id: str (optional) The ID of the cluster policy used to create the cluster if applicable. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED disks. :param runtime_engine: :class:`RuntimeEngine` (optional) Determines the cluster's runtime engine, either standard or Photon. @@ -10824,6 +11026,9 @@ def edit( SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks. :param use_ml_runtime: bool (optional) This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -10883,6 +11088,8 @@ def edit( body["num_workers"] = num_workers if policy_id is not None: body["policy_id"] = policy_id + if remote_disk_throughput is not None: + body["remote_disk_throughput"] = remote_disk_throughput if runtime_engine is not None: body["runtime_engine"] = runtime_engine.value if single_user_name is not None: @@ -10895,6 +11102,8 @@ def edit( body["spark_version"] = spark_version if ssh_public_keys is not None: body["ssh_public_keys"] = [v for v in ssh_public_keys] + if total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = total_initial_remote_disk_size if use_ml_runtime is not None: body["use_ml_runtime"] = use_ml_runtime if workload_type is not None: @@ -10936,11 +11145,13 @@ def edit_and_wait( node_type_id: Optional[str] = None, num_workers: Optional[int] = None, policy_id: Optional[str] = None, + remote_disk_throughput: Optional[int] = None, runtime_engine: Optional[RuntimeEngine] = None, single_user_name: Optional[str] = None, spark_conf: Optional[Dict[str, str]] = None, spark_env_vars: Optional[Dict[str, str]] = None, ssh_public_keys: Optional[List[str]] = None, + total_initial_remote_disk_size: Optional[int] = None, use_ml_runtime: Optional[bool] = None, workload_type: Optional[WorkloadType] = None, timeout=timedelta(minutes=20), @@ -10969,12 +11180,14 @@ def edit_and_wait( node_type_id=node_type_id, num_workers=num_workers, policy_id=policy_id, + remote_disk_throughput=remote_disk_throughput, runtime_engine=runtime_engine, single_user_name=single_user_name, spark_conf=spark_conf, spark_env_vars=spark_env_vars, spark_version=spark_version, ssh_public_keys=ssh_public_keys, + total_initial_remote_disk_size=total_initial_remote_disk_size, use_ml_runtime=use_ml_runtime, workload_type=workload_type, ).result(timeout=timeout) @@ -11161,6 +11374,7 @@ def list( def list_node_types(self) -> ListNodeTypesResponse: """Returns a list of supported Spark node types. These node types can be used to launch a cluster. + :returns: :class:`ListNodeTypesResponse` """ @@ -11175,6 +11389,7 @@ def list_zones(self) -> ListAvailableZonesResponse: """Returns a list of availability zones where clusters can be created in (For example, us-west-2a). These zones can be used to launch a cluster. + :returns: :class:`ListAvailableZonesResponse` """ @@ -11335,6 +11550,7 @@ def set_permissions( def spark_versions(self) -> GetSparkVersionsResponse: """Returns the list of available Spark versions. These versions can be used to launch a cluster. + :returns: :class:`GetSparkVersionsResponse` """ @@ -11895,6 +12111,7 @@ def list(self) -> Iterator[GlobalInitScriptDetails]: but **not** the script contents. To retrieve the contents of a script, use the [get a global init script](:method:globalinitscripts/get) operation. + :returns: Iterator over :class:`GlobalInitScriptDetails` """ @@ -11985,6 +12202,8 @@ def create( min_idle_instances: Optional[int] = None, preloaded_docker_images: Optional[List[DockerImage]] = None, preloaded_spark_versions: Optional[List[str]] = None, + remote_disk_throughput: Optional[int] = None, + total_initial_remote_disk_size: Optional[int] = None, ) -> CreateInstancePoolResponse: """Creates a new instance pool using idle and ready-to-use cloud instances. @@ -12034,6 +12253,12 @@ def create( A list containing at most one preloaded Spark image version for the pool. Pool-backed clusters started with the preloaded Spark version will start faster. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED types. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types. :returns: :class:`CreateInstancePoolResponse` """ @@ -12064,6 +12289,10 @@ def create( body["preloaded_docker_images"] = [v.as_dict() for v in preloaded_docker_images] if preloaded_spark_versions is not None: body["preloaded_spark_versions"] = [v for v in preloaded_spark_versions] + if remote_disk_throughput is not None: + body["remote_disk_throughput"] = remote_disk_throughput + if total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = total_initial_remote_disk_size headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -12100,6 +12329,8 @@ def edit( idle_instance_autotermination_minutes: Optional[int] = None, max_capacity: Optional[int] = None, min_idle_instances: Optional[int] = None, + remote_disk_throughput: Optional[int] = None, + total_initial_remote_disk_size: Optional[int] = None, ): """Modifies the configuration of an existing instance pool. @@ -12130,6 +12361,12 @@ def edit( upsize requests. :param min_idle_instances: int (optional) Minimum number of idle instances to keep in the instance pool + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED types. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types. """ @@ -12148,6 +12385,10 @@ def edit( body["min_idle_instances"] = min_idle_instances if node_type_id is not None: body["node_type_id"] = node_type_id + if remote_disk_throughput is not None: + body["remote_disk_throughput"] = remote_disk_throughput + if total_initial_remote_disk_size is not None: + body["total_initial_remote_disk_size"] = total_initial_remote_disk_size headers = { "Accept": "application/json", "Content-Type": "application/json", @@ -12212,6 +12453,7 @@ def get_permissions(self, instance_pool_id: str) -> InstancePoolPermissions: def list(self) -> Iterator[InstancePoolAndStats]: """Gets a list of instance pools with their statistics. + :returns: Iterator over :class:`InstancePoolAndStats` """ @@ -12393,6 +12635,7 @@ def list(self) -> Iterator[InstanceProfile]: This API is available to all users. + :returns: Iterator over :class:`InstanceProfile` """ @@ -12448,6 +12691,7 @@ def all_cluster_statuses(self) -> Iterator[ClusterLibraryStatuses]: """Get the status of all libraries on all clusters. A status is returned for all libraries installed on this cluster via the API or the libraries UI. + :returns: Iterator over :class:`ClusterLibraryStatuses` """ diff --git a/databricks/sdk/service/iam.py b/databricks/sdk/service/iam.py index a0f21da6e..45208bbcd 100755 --- a/databricks/sdk/service/iam.py +++ b/databricks/sdk/service/iam.py @@ -3264,6 +3264,7 @@ def __init__(self, api_client): def me(self) -> User: """Get details about the current method caller's identity. + :returns: :class:`User` """ @@ -4147,6 +4148,7 @@ def get( def get_permission_levels(self) -> GetPasswordPermissionLevelsResponse: """Gets the permission levels that a user can have on an object. + :returns: :class:`GetPasswordPermissionLevelsResponse` """ @@ -4160,6 +4162,7 @@ def get_permission_levels(self) -> GetPasswordPermissionLevelsResponse: def get_permissions(self) -> PasswordPermissions: """Gets the permissions of all passwords. Passwords can inherit permissions from their root object. + :returns: :class:`PasswordPermissions` """ diff --git a/databricks/sdk/service/jobs.py b/databricks/sdk/service/jobs.py index 37099ef24..5a75f3a00 100755 --- a/databricks/sdk/service/jobs.py +++ b/databricks/sdk/service/jobs.py @@ -260,9 +260,10 @@ class BaseRun: through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: - Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually - restart a continuous job run.""" + Indicates a run that is triggered by a file arrival. * `CONTINUOUS`: Indicates a run that is + triggered by a continuous job. * `TABLE`: Indicates a run that is triggered by a table update. * + `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. + * `MODEL`: Indicates a run that is triggered by a model update.""" trigger_info: Optional[TriggerInfo] = None """Additional details about what triggered the run""" @@ -5333,9 +5334,10 @@ class Run: through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: - Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually - restart a continuous job run.""" + Indicates a run that is triggered by a file arrival. * `CONTINUOUS`: Indicates a run that is + triggered by a continuous job. * `TABLE`: Indicates a run that is triggered by a table update. * + `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. + * `MODEL`: Indicates a run that is triggered by a model update.""" trigger_info: Optional[TriggerInfo] = None """Additional details about what triggered the run""" @@ -9154,10 +9156,13 @@ class TriggerType(Enum): through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: - Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is - triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually - restart a continuous job run.""" + Indicates a run that is triggered by a file arrival. * `CONTINUOUS`: Indicates a run that is + triggered by a continuous job. * `TABLE`: Indicates a run that is triggered by a table update. * + `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. + * `MODEL`: Indicates a run that is triggered by a model update.""" + CONTINUOUS = "CONTINUOUS" + CONTINUOUS_RESTART = "CONTINUOUS_RESTART" FILE_ARRIVAL = "FILE_ARRIVAL" ONE_TIME = "ONE_TIME" PERIODIC = "PERIODIC" diff --git a/databricks/sdk/service/marketplace.py b/databricks/sdk/service/marketplace.py index 9ac4c153a..939bf93f1 100755 --- a/databricks/sdk/service/marketplace.py +++ b/databricks/sdk/service/marketplace.py @@ -4687,6 +4687,7 @@ def create(self) -> ProviderAnalyticsDashboard: """Create provider analytics dashboard. Returns Marketplace specific `id`. Not to be confused with the Lakeview dashboard id. + :returns: :class:`ProviderAnalyticsDashboard` """ @@ -4700,6 +4701,7 @@ def create(self) -> ProviderAnalyticsDashboard: def get(self) -> ListProviderAnalyticsDashboardResponse: """Get provider analytics dashboard. + :returns: :class:`ListProviderAnalyticsDashboardResponse` """ @@ -4713,6 +4715,7 @@ def get(self) -> ListProviderAnalyticsDashboardResponse: def get_latest_version(self) -> GetLatestVersionProviderAnalyticsDashboardResponse: """Get latest version of provider analytics dashboard. + :returns: :class:`GetLatestVersionProviderAnalyticsDashboardResponse` """ diff --git a/databricks/sdk/service/provisioning.py b/databricks/sdk/service/provisioning.py index 12a5d61fa..bdae323fb 100755 --- a/databricks/sdk/service/provisioning.py +++ b/databricks/sdk/service/provisioning.py @@ -2397,6 +2397,7 @@ def get(self, credentials_id: str) -> Credential: def list(self) -> Iterator[Credential]: """Gets all Databricks credential configurations associated with an account specified by ID. + :returns: Iterator over :class:`Credential` """ @@ -2534,6 +2535,7 @@ def list(self) -> Iterator[CustomerManagedKey]: This operation is available only if your account is on the E2 version of the platform. + :returns: Iterator over :class:`CustomerManagedKey` """ @@ -2647,6 +2649,7 @@ def list(self) -> Iterator[Network]: This operation is available only if your account is on the E2 version of the platform. + :returns: Iterator over :class:`Network` """ @@ -2791,6 +2794,7 @@ def get(self, private_access_settings_id: str) -> PrivateAccessSettings: def list(self) -> Iterator[PrivateAccessSettings]: """Gets a list of all private access settings objects for an account, specified by ID. + :returns: Iterator over :class:`PrivateAccessSettings` """ @@ -2970,6 +2974,7 @@ def get(self, storage_configuration_id: str) -> StorageConfiguration: def list(self) -> Iterator[StorageConfiguration]: """Gets a list of all Databricks storage configurations for your account, specified by ID. + :returns: Iterator over :class:`StorageConfiguration` """ @@ -3091,6 +3096,7 @@ def list(self) -> Iterator[VpcEndpoint]: [Databricks article about PrivateLink]: https://docs.databricks.com/administration-guide/cloud-configurations/aws/privatelink.html + :returns: Iterator over :class:`VpcEndpoint` """ @@ -3414,6 +3420,7 @@ def list(self) -> Iterator[Workspace]: This operation is available only if your account is on the E2 version of the platform or on a select custom plan that allows multiple workspaces per account. + :returns: Iterator over :class:`Workspace` """ diff --git a/databricks/sdk/service/serving.py b/databricks/sdk/service/serving.py index 23d7db400..d0bd2dcdd 100755 --- a/databricks/sdk/service/serving.py +++ b/databricks/sdk/service/serving.py @@ -4822,6 +4822,7 @@ def http_request( def list(self) -> Iterator[ServingEndpoint]: """Get all serving endpoints. + :returns: Iterator over :class:`ServingEndpoint` """ diff --git a/databricks/sdk/service/settings.py b/databricks/sdk/service/settings.py index 6d1318a22..b6463fddb 100755 --- a/databricks/sdk/service/settings.py +++ b/databricks/sdk/service/settings.py @@ -6827,6 +6827,7 @@ def get(self, ip_access_list_id: str) -> GetIpAccessListResponse: def list(self) -> Iterator[IpAccessListInfo]: """Gets all IP access lists for the specified account. + :returns: Iterator over :class:`IpAccessListInfo` """ @@ -8007,6 +8008,7 @@ def __init__(self, api_client): def get_enable_export_notebook(self) -> EnableExportNotebook: """Gets the Notebook and File exporting setting. + :returns: :class:`EnableExportNotebook` """ @@ -8170,6 +8172,7 @@ def __init__(self, api_client): def get_enable_notebook_table_clipboard(self) -> EnableNotebookTableClipboard: """Gets the Results Table Clipboard features setting. + :returns: :class:`EnableNotebookTableClipboard` """ @@ -8231,6 +8234,7 @@ def __init__(self, api_client): def get_enable_results_downloading(self) -> EnableResultsDownloading: """Gets the Notebook results download setting. + :returns: :class:`EnableResultsDownloading` """ @@ -8535,6 +8539,7 @@ def get(self, ip_access_list_id: str) -> FetchIpAccessListResponse: def list(self) -> Iterator[IpAccessListInfo]: """Gets all IP access lists for the specified workspace. + :returns: Iterator over :class:`IpAccessListInfo` """ @@ -9896,6 +9901,7 @@ def get(self, token_id: str) -> GetTokenResponse: def get_permission_levels(self) -> GetTokenPermissionLevelsResponse: """Gets the permission levels that a user can have on an object. + :returns: :class:`GetTokenPermissionLevelsResponse` """ @@ -9909,6 +9915,7 @@ def get_permission_levels(self) -> GetTokenPermissionLevelsResponse: def get_permissions(self) -> TokenPermissions: """Gets the permissions of all tokens. Tokens can inherit permissions from their root object. + :returns: :class:`TokenPermissions` """ @@ -10044,6 +10051,7 @@ def delete(self, token_id: str): def list(self) -> Iterator[PublicTokenInfo]: """Lists all the valid tokens for a user-workspace pair. + :returns: Iterator over :class:`PublicTokenInfo` """ diff --git a/databricks/sdk/service/sql.py b/databricks/sdk/service/sql.py index 9e7eb240a..bbb3a2d90 100755 --- a/databricks/sdk/service/sql.py +++ b/databricks/sdk/service/sql.py @@ -3419,7 +3419,8 @@ def from_dict(cls, d: Dict[str, Any]) -> EnumValue: @dataclass class ExecuteStatementRequest: statement: str - """The SQL statement to execute. The statement can optionally be parameterized, see `parameters`.""" + """The SQL statement to execute. The statement can optionally be parameterized, see `parameters`. + The maximum query text size is 16 MiB.""" warehouse_id: str """Warehouse upon which to execute a statement. See also [What are SQL warehouses?] @@ -8914,6 +8915,7 @@ def list(self) -> Iterator[LegacyAlert]: [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + :returns: Iterator over :class:`LegacyAlert` """ @@ -9407,6 +9409,7 @@ def list(self) -> Iterator[DataSource]: [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + :returns: Iterator over :class:`DataSource` """ @@ -10261,6 +10264,7 @@ def __init__(self, api_client): def get_config(self) -> ClientConfig: """Read workspace configuration for Redash-v2. + :returns: :class:`ClientConfig` """ @@ -10394,7 +10398,8 @@ def execute_statement( """Execute a SQL statement :param statement: str - The SQL statement to execute. The statement can optionally be parameterized, see `parameters`. + The SQL statement to execute. The statement can optionally be parameterized, see `parameters`. The + maximum query text size is 16 MiB. :param warehouse_id: str Warehouse upon which to execute a statement. See also [What are SQL warehouses?] @@ -11018,6 +11023,7 @@ def get_permissions(self, warehouse_id: str) -> WarehousePermissions: def get_workspace_warehouse_config(self) -> GetWorkspaceWarehouseConfigResponse: """Gets the workspace level configuration that is shared by all SQL warehouses in a workspace. + :returns: :class:`GetWorkspaceWarehouseConfigResponse` """ diff --git a/databricks/sdk/service/workspace.py b/databricks/sdk/service/workspace.py index 556bf1fdb..a7ecb15a9 100755 --- a/databricks/sdk/service/workspace.py +++ b/databricks/sdk/service/workspace.py @@ -2443,6 +2443,7 @@ def get(self, credential_id: int) -> GetCredentialsResponse: def list(self) -> Iterator[CredentialInfo]: """Lists the calling user's Git credentials. One credential per user is supported. + :returns: Iterator over :class:`CredentialInfo` """ @@ -2943,6 +2944,7 @@ def list_scopes(self) -> Iterator[SecretScope]: Throws `PERMISSION_DENIED` if the user does not have permission to make this API call. + :returns: Iterator over :class:`SecretScope` """ diff --git a/docs/account/billing/log_delivery.rst b/docs/account/billing/log_delivery.rst index 1eefc8802..41c4df845 100644 --- a/docs/account/billing/log_delivery.rst +++ b/docs/account/billing/log_delivery.rst @@ -4,8 +4,57 @@ .. py:class:: LogDeliveryAPI - These APIs manage Log delivery configurations for this account. Log delivery configs enable you to - configure the delivery of the specified type of logs to your storage account. + These APIs manage log delivery configurations for this account. The two supported log types for this API + are _billable usage logs_ and _audit logs_. This feature is in Public Preview. This feature works with all + account ID types. + + Log delivery works with all account types. However, if your account is on the E2 version of the platform + or on a select custom plan that allows multiple workspaces per account, you can optionally configure + different storage destinations for each workspace. Log delivery status is also provided to know the latest + status of log delivery attempts. + + The high-level flow of billable usage delivery: + + 1. **Create storage**: In AWS, [create a new AWS S3 bucket] with a specific bucket policy. Using + Databricks APIs, call the Account API to create a [storage configuration object](:method:Storage/Create) + that uses the bucket name. + + 2. **Create credentials**: In AWS, create the appropriate AWS IAM role. For full details, including the + required IAM role policies and trust relationship, see [Billable usage log delivery]. Using Databricks + APIs, call the Account API to create a [credential configuration object](:method:Credentials/Create) that + uses the IAM role's ARN. + + 3. **Create log delivery configuration**: Using Databricks APIs, call the Account API to [create a log + delivery configuration](:method:LogDelivery/Create) that uses the credential and storage configuration + objects from previous steps. You can specify if the logs should include all events of that log type in + your account (_Account level_ delivery) or only events for a specific set of workspaces (_workspace level_ + delivery). Account level log delivery applies to all current and future workspaces plus account level + logs, while workspace level log delivery solely delivers logs related to the specified workspaces. You can + create multiple types of delivery configurations per account. + + For billable usage delivery: * For more information about billable usage logs, see [Billable usage log + delivery]. For the CSV schema, see the [Usage page]. * The delivery location is + `//billable-usage/csv/`, where `` is the name of the optional delivery path + prefix you set up during log delivery configuration. Files are named + `workspaceId=-usageMonth=.csv`. * All billable usage logs apply to specific + workspaces (_workspace level_ logs). You can aggregate usage for your entire account by creating an + _account level_ delivery configuration that delivers logs for all current and future workspaces in your + account. * The files are delivered daily by overwriting the month's CSV file for each workspace. + + For audit log delivery: * For more information about about audit log delivery, see [Audit log delivery], + which includes information about the used JSON schema. * The delivery location is + `//workspaceId=/date=/auditlogs_.json`. + Files may get overwritten with the same content multiple times to achieve exactly-once delivery. * If the + audit log delivery configuration included specific workspace IDs, only _workspace-level_ audit logs for + those workspaces are delivered. If the log delivery configuration applies to the entire account (_account + level_ delivery configuration), the audit log delivery includes workspace-level audit logs for all + workspaces in the account as well as account-level audit logs. See [Audit log delivery] for details. * + Auditable events are typically available in logs within 15 minutes. + + [Audit log delivery]: https://docs.databricks.com/administration-guide/account-settings/audit-logs.html + [Billable usage log delivery]: https://docs.databricks.com/administration-guide/account-settings/billable-usage-delivery.html + [Usage page]: https://docs.databricks.com/administration-guide/account-settings/usage.html + [create a new AWS S3 bucket]: https://docs.databricks.com/administration-guide/account-api/aws-storage.html .. py:method:: create(log_delivery_configuration: CreateLogDeliveryConfigurationParams) -> WrappedLogDeliveryConfiguration diff --git a/docs/account/catalog/metastores.rst b/docs/account/catalog/metastores.rst index 3cae5dd31..bb34cf6eb 100644 --- a/docs/account/catalog/metastores.rst +++ b/docs/account/catalog/metastores.rst @@ -42,6 +42,7 @@ Gets all Unity Catalog metastores associated with an account specified by ID. + :returns: Iterator over :class:`MetastoreInfo` diff --git a/docs/account/iam/workspace_assignment.rst b/docs/account/iam/workspace_assignment.rst index fa9c2ee3e..133b16f3d 100644 --- a/docs/account/iam/workspace_assignment.rst +++ b/docs/account/iam/workspace_assignment.rst @@ -43,9 +43,9 @@ a = AccountClient() - workspace_id = os.environ["DUMMY_WORKSPACE_ID"] + workspace_id = os.environ["TEST_WORKSPACE_ID"] - all = a.workspace_assignment.list(workspace_id=workspace_id) + all = a.workspace_assignment.list(list=workspace_id) Get the permission assignments for the specified Databricks account and Databricks workspace. diff --git a/docs/account/provisioning/credentials.rst b/docs/account/provisioning/credentials.rst index faf3b52ba..acb958c8c 100644 --- a/docs/account/provisioning/credentials.rst +++ b/docs/account/provisioning/credentials.rst @@ -115,5 +115,6 @@ Gets all Databricks credential configurations associated with an account specified by ID. + :returns: Iterator over :class:`Credential` \ No newline at end of file diff --git a/docs/account/provisioning/encryption_keys.rst b/docs/account/provisioning/encryption_keys.rst index f13513446..3e190c250 100644 --- a/docs/account/provisioning/encryption_keys.rst +++ b/docs/account/provisioning/encryption_keys.rst @@ -146,5 +146,6 @@ This operation is available only if your account is on the E2 version of the platform. + :returns: Iterator over :class:`CustomerManagedKey` \ No newline at end of file diff --git a/docs/account/provisioning/networks.rst b/docs/account/provisioning/networks.rst index 0b3b8f8c2..54b24ebc2 100644 --- a/docs/account/provisioning/networks.rst +++ b/docs/account/provisioning/networks.rst @@ -113,5 +113,6 @@ This operation is available only if your account is on the E2 version of the platform. + :returns: Iterator over :class:`Network` \ No newline at end of file diff --git a/docs/account/provisioning/private_access.rst b/docs/account/provisioning/private_access.rst index ca422e8b1..414ee23fd 100644 --- a/docs/account/provisioning/private_access.rst +++ b/docs/account/provisioning/private_access.rst @@ -141,6 +141,7 @@ Gets a list of all private access settings objects for an account, specified by ID. + :returns: Iterator over :class:`PrivateAccessSettings` diff --git a/docs/account/provisioning/storage.rst b/docs/account/provisioning/storage.rst index c89589282..8e89e714f 100644 --- a/docs/account/provisioning/storage.rst +++ b/docs/account/provisioning/storage.rst @@ -105,5 +105,6 @@ Gets a list of all Databricks storage configurations for your account, specified by ID. + :returns: Iterator over :class:`StorageConfiguration` \ No newline at end of file diff --git a/docs/account/provisioning/vpc_endpoints.rst b/docs/account/provisioning/vpc_endpoints.rst index 032365197..0807679f7 100644 --- a/docs/account/provisioning/vpc_endpoints.rst +++ b/docs/account/provisioning/vpc_endpoints.rst @@ -127,5 +127,6 @@ [Databricks article about PrivateLink]: https://docs.databricks.com/administration-guide/cloud-configurations/aws/privatelink.html + :returns: Iterator over :class:`VpcEndpoint` \ No newline at end of file diff --git a/docs/account/provisioning/workspaces.rst b/docs/account/provisioning/workspaces.rst index f49d58cc2..4566dd0cf 100644 --- a/docs/account/provisioning/workspaces.rst +++ b/docs/account/provisioning/workspaces.rst @@ -223,6 +223,7 @@ This operation is available only if your account is on the E2 version of the platform or on a select custom plan that allows multiple workspaces per account. + :returns: Iterator over :class:`Workspace` diff --git a/docs/account/settings/ip_access_lists.rst b/docs/account/settings/ip_access_lists.rst index 0837cd0f7..7e4b65b2e 100644 --- a/docs/account/settings/ip_access_lists.rst +++ b/docs/account/settings/ip_access_lists.rst @@ -75,6 +75,7 @@ Gets all IP access lists for the specified account. + :returns: Iterator over :class:`IpAccessListInfo` diff --git a/docs/dbdataclasses/compute.rst b/docs/dbdataclasses/compute.rst index 54b17b745..a283d6dd9 100644 --- a/docs/dbdataclasses/compute.rst +++ b/docs/dbdataclasses/compute.rst @@ -825,6 +825,9 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:attribute:: PYTHON :value: "PYTHON" + .. py:attribute:: R + :value: "R" + .. py:attribute:: SCALA :value: "SCALA" diff --git a/docs/dbdataclasses/jobs.rst b/docs/dbdataclasses/jobs.rst index af4915a6b..b1365d27c 100644 --- a/docs/dbdataclasses/jobs.rst +++ b/docs/dbdataclasses/jobs.rst @@ -1128,7 +1128,13 @@ These dataclasses are used in the SDK to represent API requests and responses fo .. py:class:: TriggerType The type of trigger that fired this run. - * `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler. * `ONE_TIME`: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `TABLE`: Indicates a run that is triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. + * `PERIODIC`: Schedules that periodically trigger runs, such as a cron scheduler. * `ONE_TIME`: One time triggers that fire a single run. This occurs you triggered a single run on demand through the UI or the API. * `RETRY`: Indicates a run that is triggered as a retry of a previously failed run. This occurs when you request to re-run the job in case of failures. * `RUN_JOB_TASK`: Indicates a run that is triggered using a Run Job task. * `FILE_ARRIVAL`: Indicates a run that is triggered by a file arrival. * `CONTINUOUS`: Indicates a run that is triggered by a continuous job. * `TABLE`: Indicates a run that is triggered by a table update. * `CONTINUOUS_RESTART`: Indicates a run created by user to manually restart a continuous job run. * `MODEL`: Indicates a run that is triggered by a model update. + + .. py:attribute:: CONTINUOUS + :value: "CONTINUOUS" + + .. py:attribute:: CONTINUOUS_RESTART + :value: "CONTINUOUS_RESTART" .. py:attribute:: FILE_ARRIVAL :value: "FILE_ARRIVAL" diff --git a/docs/workspace/catalog/external_locations.rst b/docs/workspace/catalog/external_locations.rst index c3b5217a1..e12792487 100644 --- a/docs/workspace/catalog/external_locations.rst +++ b/docs/workspace/catalog/external_locations.rst @@ -107,20 +107,20 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), + url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', ) - _ = w.external_locations.get(name=created.name) + _ = w.external_locations.get(get=created.name) # cleanup - w.storage_credentials.delete(name=credential.name) - w.external_locations.delete(name=created.name) + w.storage_credentials.delete(delete=credential.name) + w.external_locations.delete(delete=created.name) Gets an external location from the metastore. The caller must be either a metastore admin, the owner of the external location, or a user that has some privilege on the external location. @@ -142,10 +142,11 @@ .. code-block:: from databricks.sdk import WorkspaceClient + from databricks.sdk.service import catalog w = WorkspaceClient() - all = w.external_locations.list() + all = w.external_locations.list(catalog.ListExternalLocationsRequest()) Gets an array of external locations (__ExternalLocationInfo__ objects) from the metastore. The caller must be a metastore admin, the owner of the external location, or a user that has some privilege on @@ -182,24 +183,24 @@ credential = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) created = w.external_locations.create( name=f"sdk-{time.time_ns()}", credential_name=credential.name, - url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', + url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), ) _ = w.external_locations.update( name=created.name, credential_name=credential.name, - url=f's3://{os.environ["TEST_BUCKET"]}/sdk-{time.time_ns()}', + url="s3://%s/%s" % (os.environ["TEST_BUCKET"], f"sdk-{time.time_ns()}"), ) # cleanup - w.storage_credentials.delete(delete=credential.name) - w.external_locations.delete(delete=created.name) + w.storage_credentials.delete(name=credential.name) + w.external_locations.delete(name=created.name) Updates an external location in the metastore. The caller must be the owner of the external location, or be a metastore admin. In the second case, the admin can only update the name of the external diff --git a/docs/workspace/catalog/metastores.rst b/docs/workspace/catalog/metastores.rst index 35fdf0d41..28636acf9 100644 --- a/docs/workspace/catalog/metastores.rst +++ b/docs/workspace/catalog/metastores.rst @@ -109,6 +109,7 @@ Gets the metastore assignment for the workspace being accessed. + :returns: :class:`MetastoreAssignment` @@ -203,6 +204,7 @@ Gets information about a metastore. This summary includes the storage credential, the cloud vendor, the cloud region, and the global metastore ID. + :returns: :class:`GetMetastoreSummaryResponse` diff --git a/docs/workspace/catalog/storage_credentials.rst b/docs/workspace/catalog/storage_credentials.rst index 194069200..5fe0bb70f 100644 --- a/docs/workspace/catalog/storage_credentials.rst +++ b/docs/workspace/catalog/storage_credentials.rst @@ -32,11 +32,11 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) # cleanup - w.storage_credentials.delete(name=created.name) + w.storage_credentials.delete(delete=created.name) Creates a new storage credential. @@ -98,13 +98,13 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) - by_name = w.storage_credentials.get(get=created.name) + by_name = w.storage_credentials.get(name=created.name) # cleanup - w.storage_credentials.delete(delete=created.name) + w.storage_credentials.delete(name=created.name) Gets a storage credential from the metastore. The caller must be a metastore admin, the owner of the storage credential, or have some permission on the storage credential. @@ -162,17 +162,17 @@ created = w.storage_credentials.create( name=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) _ = w.storage_credentials.update( name=created.name, comment=f"sdk-{time.time_ns()}", - aws_iam_role=catalog.AwsIamRoleRequest(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), + aws_iam_role=catalog.AwsIamRole(role_arn=os.environ["TEST_METASTORE_DATA_ACCESS_ARN"]), ) # cleanup - w.storage_credentials.delete(name=created.name) + w.storage_credentials.delete(delete=created.name) Updates a storage credential on the metastore. diff --git a/docs/workspace/catalog/tables.rst b/docs/workspace/catalog/tables.rst index 9632dc0d8..fdb5164f0 100644 --- a/docs/workspace/catalog/tables.rst +++ b/docs/workspace/catalog/tables.rst @@ -117,7 +117,7 @@ created_schema = w.schemas.create(name=f"sdk-{time.time_ns()}", catalog_name=created_catalog.name) - summaries = w.tables.list_summaries(catalog_name=created_catalog.name, schema_name_pattern=created_schema.name) + all_tables = w.tables.list(catalog_name=created_catalog.name, schema_name=created_schema.name) # cleanup w.schemas.delete(full_name=created_schema.full_name) diff --git a/docs/workspace/compute/clusters.rst b/docs/workspace/compute/clusters.rst index 5e1abee9f..cf535e30d 100644 --- a/docs/workspace/compute/clusters.rst +++ b/docs/workspace/compute/clusters.rst @@ -70,7 +70,7 @@ - .. py:method:: create(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] + .. py:method:: create(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], remote_disk_throughput: Optional[int], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], total_initial_remote_disk_size: Optional[int], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] Usage: @@ -239,6 +239,9 @@ provisioned. :param policy_id: str (optional) The ID of the cluster policy used to create the cluster if applicable. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED disks. :param runtime_engine: :class:`RuntimeEngine` (optional) Determines the cluster's runtime engine, either standard or Photon. @@ -269,6 +272,9 @@ SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks. :param use_ml_runtime: bool (optional) This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -282,7 +288,7 @@ See :method:wait_get_cluster_running for more details. - .. py:method:: create_and_wait(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails + .. py:method:: create_and_wait(spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], clone_from: Optional[CloneCluster], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], remote_disk_throughput: Optional[int], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], total_initial_remote_disk_size: Optional[int], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails .. py:method:: delete(cluster_id: str) -> Wait[ClusterDetails] @@ -331,7 +337,7 @@ .. py:method:: delete_and_wait(cluster_id: str, timeout: datetime.timedelta = 0:20:00) -> ClusterDetails - .. py:method:: edit(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] + .. py:method:: edit(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], remote_disk_throughput: Optional[int], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], total_initial_remote_disk_size: Optional[int], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType]]) -> Wait[ClusterDetails] Usage: @@ -506,6 +512,9 @@ provisioned. :param policy_id: str (optional) The ID of the cluster policy used to create the cluster if applicable. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED disks. :param runtime_engine: :class:`RuntimeEngine` (optional) Determines the cluster's runtime engine, either standard or Photon. @@ -536,6 +545,9 @@ SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name `ubuntu` on port `2200`. Up to 10 keys can be specified. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED disks. :param use_ml_runtime: bool (optional) This field can only be used when `kind = CLASSIC_PREVIEW`. @@ -549,7 +561,7 @@ See :method:wait_get_cluster_running for more details. - .. py:method:: edit_and_wait(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails + .. py:method:: edit_and_wait(cluster_id: str, spark_version: str [, apply_policy_default_values: Optional[bool], autoscale: Optional[AutoScale], autotermination_minutes: Optional[int], aws_attributes: Optional[AwsAttributes], azure_attributes: Optional[AzureAttributes], cluster_log_conf: Optional[ClusterLogConf], cluster_name: Optional[str], custom_tags: Optional[Dict[str, str]], data_security_mode: Optional[DataSecurityMode], docker_image: Optional[DockerImage], driver_instance_pool_id: Optional[str], driver_node_type_id: Optional[str], enable_elastic_disk: Optional[bool], enable_local_disk_encryption: Optional[bool], gcp_attributes: Optional[GcpAttributes], init_scripts: Optional[List[InitScriptInfo]], instance_pool_id: Optional[str], is_single_node: Optional[bool], kind: Optional[Kind], node_type_id: Optional[str], num_workers: Optional[int], policy_id: Optional[str], remote_disk_throughput: Optional[int], runtime_engine: Optional[RuntimeEngine], single_user_name: Optional[str], spark_conf: Optional[Dict[str, str]], spark_env_vars: Optional[Dict[str, str]], ssh_public_keys: Optional[List[str]], total_initial_remote_disk_size: Optional[int], use_ml_runtime: Optional[bool], workload_type: Optional[WorkloadType], timeout: datetime.timedelta = 0:20:00]) -> ClusterDetails .. py:method:: ensure_cluster_is_running(cluster_id: str) @@ -749,6 +761,7 @@ Returns a list of supported Spark node types. These node types can be used to launch a cluster. + :returns: :class:`ListNodeTypesResponse` @@ -757,6 +770,7 @@ Returns a list of availability zones where clusters can be created in (For example, us-west-2a). These zones can be used to launch a cluster. + :returns: :class:`ListAvailableZonesResponse` @@ -993,6 +1007,7 @@ Returns the list of available Spark versions. These versions can be used to launch a cluster. + :returns: :class:`GetSparkVersionsResponse` diff --git a/docs/workspace/compute/global_init_scripts.rst b/docs/workspace/compute/global_init_scripts.rst index 78854102d..eb01c9f63 100644 --- a/docs/workspace/compute/global_init_scripts.rst +++ b/docs/workspace/compute/global_init_scripts.rst @@ -119,6 +119,7 @@ but **not** the script contents. To retrieve the contents of a script, use the [get a global init script](:method:globalinitscripts/get) operation. + :returns: Iterator over :class:`GlobalInitScriptDetails` diff --git a/docs/workspace/compute/instance_pools.rst b/docs/workspace/compute/instance_pools.rst index e5c966895..8be26a2d6 100644 --- a/docs/workspace/compute/instance_pools.rst +++ b/docs/workspace/compute/instance_pools.rst @@ -19,7 +19,7 @@ Databricks does not charge DBUs while instances are idle in the pool. Instance provider billing does apply. See pricing. - .. py:method:: create(instance_pool_name: str, node_type_id: str [, aws_attributes: Optional[InstancePoolAwsAttributes], azure_attributes: Optional[InstancePoolAzureAttributes], custom_tags: Optional[Dict[str, str]], disk_spec: Optional[DiskSpec], enable_elastic_disk: Optional[bool], gcp_attributes: Optional[InstancePoolGcpAttributes], idle_instance_autotermination_minutes: Optional[int], max_capacity: Optional[int], min_idle_instances: Optional[int], preloaded_docker_images: Optional[List[DockerImage]], preloaded_spark_versions: Optional[List[str]]]) -> CreateInstancePoolResponse + .. py:method:: create(instance_pool_name: str, node_type_id: str [, aws_attributes: Optional[InstancePoolAwsAttributes], azure_attributes: Optional[InstancePoolAzureAttributes], custom_tags: Optional[Dict[str, str]], disk_spec: Optional[DiskSpec], enable_elastic_disk: Optional[bool], gcp_attributes: Optional[InstancePoolGcpAttributes], idle_instance_autotermination_minutes: Optional[int], max_capacity: Optional[int], min_idle_instances: Optional[int], preloaded_docker_images: Optional[List[DockerImage]], preloaded_spark_versions: Optional[List[str]], remote_disk_throughput: Optional[int], total_initial_remote_disk_size: Optional[int]]) -> CreateInstancePoolResponse Usage: @@ -87,6 +87,12 @@ A list containing at most one preloaded Spark image version for the pool. Pool-backed clusters started with the preloaded Spark version will start faster. A list of available Spark versions can be retrieved by using the :method:clusters/sparkVersions API call. + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED types. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types. :returns: :class:`CreateInstancePoolResponse` @@ -101,7 +107,7 @@ - .. py:method:: edit(instance_pool_id: str, instance_pool_name: str, node_type_id: str [, custom_tags: Optional[Dict[str, str]], idle_instance_autotermination_minutes: Optional[int], max_capacity: Optional[int], min_idle_instances: Optional[int]]) + .. py:method:: edit(instance_pool_id: str, instance_pool_name: str, node_type_id: str [, custom_tags: Optional[Dict[str, str]], idle_instance_autotermination_minutes: Optional[int], max_capacity: Optional[int], min_idle_instances: Optional[int], remote_disk_throughput: Optional[int], total_initial_remote_disk_size: Optional[int]]) Usage: @@ -156,6 +162,12 @@ upsize requests. :param min_idle_instances: int (optional) Minimum number of idle instances to keep in the instance pool + :param remote_disk_throughput: int (optional) + If set, what the configurable throughput (in Mb/s) for the remote disk is. Currently only supported + for GCP HYPERDISK_BALANCED types. + :param total_initial_remote_disk_size: int (optional) + If set, what the total initial volume size (in GB) of the remote disks should be. Currently only + supported for GCP HYPERDISK_BALANCED types. @@ -226,6 +238,7 @@ Gets a list of instance pools with their statistics. + :returns: Iterator over :class:`InstancePoolAndStats` diff --git a/docs/workspace/compute/instance_profiles.rst b/docs/workspace/compute/instance_profiles.rst index 2af2d4b68..7a5192bd5 100644 --- a/docs/workspace/compute/instance_profiles.rst +++ b/docs/workspace/compute/instance_profiles.rst @@ -127,6 +127,7 @@ This API is available to all users. + :returns: Iterator over :class:`InstanceProfile` diff --git a/docs/workspace/compute/libraries.rst b/docs/workspace/compute/libraries.rst index f11857623..dfafea7aa 100644 --- a/docs/workspace/compute/libraries.rst +++ b/docs/workspace/compute/libraries.rst @@ -23,6 +23,7 @@ Get the status of all libraries on all clusters. A status is returned for all libraries installed on this cluster via the API or the libraries UI. + :returns: Iterator over :class:`ClusterLibraryStatuses` diff --git a/docs/workspace/iam/current_user.rst b/docs/workspace/iam/current_user.rst index 3b361aaf1..2f95213e2 100644 --- a/docs/workspace/iam/current_user.rst +++ b/docs/workspace/iam/current_user.rst @@ -17,9 +17,10 @@ w = WorkspaceClient() - me = w.current_user.me() + me2 = w.current_user.me() Get details about the current method caller's identity. + :returns: :class:`User` \ No newline at end of file diff --git a/docs/workspace/iam/permissions.rst b/docs/workspace/iam/permissions.rst index 4e55da96e..f166ecc2d 100644 --- a/docs/workspace/iam/permissions.rst +++ b/docs/workspace/iam/permissions.rst @@ -44,7 +44,7 @@ obj = w.workspace.get_status(path=notebook_path) - _ = w.permissions.get(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) + levels = w.permissions.get_permission_levels(request_object_type="notebooks", request_object_id="%d" % (obj.object_id)) Gets the permissions of an object. Objects can inherit permissions from their parent objects or root object. diff --git a/docs/workspace/iam/service_principals.rst b/docs/workspace/iam/service_principals.rst index 0d0d447b9..ce8978afb 100644 --- a/docs/workspace/iam/service_principals.rst +++ b/docs/workspace/iam/service_principals.rst @@ -20,13 +20,19 @@ import time from databricks.sdk import WorkspaceClient + from databricks.sdk.service import iam w = WorkspaceClient() - created = w.service_principals.create(display_name=f"sdk-{time.time_ns()}") + groups = w.groups.group_display_name_to_id_map(iam.ListGroupsRequest()) + + spn = w.service_principals.create( + display_name=f"sdk-{time.time_ns()}", + groups=[iam.ComplexValue(value=groups["admins"])], + ) # cleanup - w.service_principals.delete(id=created.id) + w.service_principals.delete(id=spn.id) Creates a new service principal in the Databricks workspace. diff --git a/docs/workspace/iam/users.rst b/docs/workspace/iam/users.rst index b48b26b02..fbee85661 100644 --- a/docs/workspace/iam/users.rst +++ b/docs/workspace/iam/users.rst @@ -78,12 +78,9 @@ w = WorkspaceClient() - user = w.users.create( - display_name=f"sdk-{time.time_ns()}", - user_name=f"sdk-{time.time_ns()}@example.com", - ) + other_owner = w.users.create(user_name=f"sdk-{time.time_ns()}@example.com") - w.users.delete(id=user.id) + w.users.delete(id=other_owner.id) Deletes a user. Deleting a user from a Databricks workspace also removes objects associated with the user. @@ -146,6 +143,7 @@ Gets the permission levels that a user can have on an object. + :returns: :class:`GetPasswordPermissionLevelsResponse` @@ -153,6 +151,7 @@ Gets the permissions of all passwords. Passwords can inherit permissions from their root object. + :returns: :class:`PasswordPermissions` diff --git a/docs/workspace/marketplace/provider_provider_analytics_dashboards.rst b/docs/workspace/marketplace/provider_provider_analytics_dashboards.rst index bd873c551..2e6878988 100644 --- a/docs/workspace/marketplace/provider_provider_analytics_dashboards.rst +++ b/docs/workspace/marketplace/provider_provider_analytics_dashboards.rst @@ -11,6 +11,7 @@ Create provider analytics dashboard. Returns Marketplace specific `id`. Not to be confused with the Lakeview dashboard id. + :returns: :class:`ProviderAnalyticsDashboard` @@ -18,6 +19,7 @@ Get provider analytics dashboard. + :returns: :class:`ListProviderAnalyticsDashboardResponse` @@ -25,6 +27,7 @@ Get latest version of provider analytics dashboard. + :returns: :class:`GetLatestVersionProviderAnalyticsDashboardResponse` diff --git a/docs/workspace/ml/model_registry.rst b/docs/workspace/ml/model_registry.rst index 0146f7902..79c1addbf 100644 --- a/docs/workspace/ml/model_registry.rst +++ b/docs/workspace/ml/model_registry.rst @@ -91,8 +91,6 @@ w = WorkspaceClient() model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - - mv = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") Creates a new registered model with the name specified in the request body. @@ -713,13 +711,14 @@ w = WorkspaceClient() - created = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") + model = w.model_registry.create_model(name=f"sdk-{time.time_ns()}") - model = w.model_registry.get_model(name=created.registered_model.name) + created = w.model_registry.create_model_version(name=model.registered_model.name, source="dbfs:/tmp") - w.model_registry.update_model( - name=model.registered_model_databricks.name, + w.model_registry.update_model_version( description=f"sdk-{time.time_ns()}", + name=created.model_version.name, + version=created.model_version.version, ) Updates a registered model. diff --git a/docs/workspace/serving/serving_endpoints.rst b/docs/workspace/serving/serving_endpoints.rst index c6efae531..404837e10 100644 --- a/docs/workspace/serving/serving_endpoints.rst +++ b/docs/workspace/serving/serving_endpoints.rst @@ -174,6 +174,7 @@ Get all serving endpoints. + :returns: Iterator over :class:`ServingEndpoint` diff --git a/docs/workspace/settings/enable_export_notebook.rst b/docs/workspace/settings/enable_export_notebook.rst index 9e0bec346..8d7df57d5 100644 --- a/docs/workspace/settings/enable_export_notebook.rst +++ b/docs/workspace/settings/enable_export_notebook.rst @@ -11,6 +11,7 @@ Gets the Notebook and File exporting setting. + :returns: :class:`EnableExportNotebook` diff --git a/docs/workspace/settings/enable_notebook_table_clipboard.rst b/docs/workspace/settings/enable_notebook_table_clipboard.rst index de65536da..c3e7b656e 100644 --- a/docs/workspace/settings/enable_notebook_table_clipboard.rst +++ b/docs/workspace/settings/enable_notebook_table_clipboard.rst @@ -11,6 +11,7 @@ Gets the Results Table Clipboard features setting. + :returns: :class:`EnableNotebookTableClipboard` diff --git a/docs/workspace/settings/enable_results_downloading.rst b/docs/workspace/settings/enable_results_downloading.rst index 799f8f04d..b50d45b64 100644 --- a/docs/workspace/settings/enable_results_downloading.rst +++ b/docs/workspace/settings/enable_results_downloading.rst @@ -10,6 +10,7 @@ Gets the Notebook results download setting. + :returns: :class:`EnableResultsDownloading` diff --git a/docs/workspace/settings/ip_access_lists.rst b/docs/workspace/settings/ip_access_lists.rst index 09fbc071f..dd51b8d75 100644 --- a/docs/workspace/settings/ip_access_lists.rst +++ b/docs/workspace/settings/ip_access_lists.rst @@ -130,6 +130,7 @@ Gets all IP access lists for the specified workspace. + :returns: Iterator over :class:`IpAccessListInfo` diff --git a/docs/workspace/settings/token_management.rst b/docs/workspace/settings/token_management.rst index a5a22552c..ee0e831b8 100644 --- a/docs/workspace/settings/token_management.rst +++ b/docs/workspace/settings/token_management.rst @@ -97,6 +97,7 @@ Gets the permission levels that a user can have on an object. + :returns: :class:`GetTokenPermissionLevelsResponse` @@ -104,6 +105,7 @@ Gets the permissions of all tokens. Tokens can inherit permissions from their root object. + :returns: :class:`TokenPermissions` diff --git a/docs/workspace/settings/tokens.rst b/docs/workspace/settings/tokens.rst index c1067d0f7..18e7161cc 100644 --- a/docs/workspace/settings/tokens.rst +++ b/docs/workspace/settings/tokens.rst @@ -66,5 +66,6 @@ Lists all the valid tokens for a user-workspace pair. + :returns: Iterator over :class:`PublicTokenInfo` \ No newline at end of file diff --git a/docs/workspace/sql/alerts_legacy.rst b/docs/workspace/sql/alerts_legacy.rst index d9662f456..199c1dba0 100644 --- a/docs/workspace/sql/alerts_legacy.rst +++ b/docs/workspace/sql/alerts_legacy.rst @@ -77,6 +77,7 @@ [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + :returns: Iterator over :class:`LegacyAlert` diff --git a/docs/workspace/sql/data_sources.rst b/docs/workspace/sql/data_sources.rst index 01ec16bdc..6e85f43ed 100644 --- a/docs/workspace/sql/data_sources.rst +++ b/docs/workspace/sql/data_sources.rst @@ -38,5 +38,6 @@ [Learn more]: https://docs.databricks.com/en/sql/dbsql-api-latest.html + :returns: Iterator over :class:`DataSource` \ No newline at end of file diff --git a/docs/workspace/sql/queries.rst b/docs/workspace/sql/queries.rst index 0dfb63fbf..f0081b3f2 100644 --- a/docs/workspace/sql/queries.rst +++ b/docs/workspace/sql/queries.rst @@ -29,7 +29,7 @@ display_name=f"sdk-{time.time_ns()}", warehouse_id=srcs[0].warehouse_id, description="test query from Go SDK", - query_text="SELECT 1", + query_text="SHOW TABLES", ) ) diff --git a/docs/workspace/sql/redash_config.rst b/docs/workspace/sql/redash_config.rst index cee23c1bd..2f29b4d75 100644 --- a/docs/workspace/sql/redash_config.rst +++ b/docs/workspace/sql/redash_config.rst @@ -10,5 +10,6 @@ Read workspace configuration for Redash-v2. + :returns: :class:`ClientConfig` \ No newline at end of file diff --git a/docs/workspace/sql/statement_execution.rst b/docs/workspace/sql/statement_execution.rst index 19bca1fdb..12bd8ba7a 100644 --- a/docs/workspace/sql/statement_execution.rst +++ b/docs/workspace/sql/statement_execution.rst @@ -106,7 +106,8 @@ Execute a SQL statement :param statement: str - The SQL statement to execute. The statement can optionally be parameterized, see `parameters`. + The SQL statement to execute. The statement can optionally be parameterized, see `parameters`. The + maximum query text size is 16 MiB. :param warehouse_id: str Warehouse upon which to execute a statement. See also [What are SQL warehouses?] diff --git a/docs/workspace/sql/warehouses.rst b/docs/workspace/sql/warehouses.rst index 5a88cdbf3..51ab7c086 100644 --- a/docs/workspace/sql/warehouses.rst +++ b/docs/workspace/sql/warehouses.rst @@ -274,6 +274,7 @@ Gets the workspace level configuration that is shared by all SQL warehouses in a workspace. + :returns: :class:`GetWorkspaceWarehouseConfigResponse` diff --git a/docs/workspace/workspace/git_credentials.rst b/docs/workspace/workspace/git_credentials.rst index eda0831be..3569abd56 100644 --- a/docs/workspace/workspace/git_credentials.rst +++ b/docs/workspace/workspace/git_credentials.rst @@ -100,6 +100,7 @@ Lists the calling user's Git credentials. One credential per user is supported. + :returns: Iterator over :class:`CredentialInfo` diff --git a/docs/workspace/workspace/secrets.rst b/docs/workspace/workspace/secrets.rst index a92042354..239b3f003 100644 --- a/docs/workspace/workspace/secrets.rst +++ b/docs/workspace/workspace/secrets.rst @@ -186,6 +186,7 @@ Throws `PERMISSION_DENIED` if the user does not have permission to make this API call. + :returns: Iterator over :class:`SecretScope` diff --git a/docs/workspace/workspace/workspace.rst b/docs/workspace/workspace/workspace.rst index 2c369968e..4fba581e8 100644 --- a/docs/workspace/workspace/workspace.rst +++ b/docs/workspace/workspace/workspace.rst @@ -223,16 +223,14 @@ .. code-block:: - import os - import time - from databricks.sdk import WorkspaceClient w = WorkspaceClient() - notebook = f"/Users/{w.current_user.me().user_name}/sdk-{time.time_ns()}" - - objects = w.workspace.list(path=os.path.dirname(notebook)) + names = [] + for i in w.workspace.list(f"/Users/{w.current_user.me().user_name}", recursive=True): + names.append(i.path) + assert len(names) > 0 List workspace objects