Merge branch 'databricks:main' into main

Author: pivoshenko

.codegen/__init__.py.tmpl

@@ -41,14 +41,14 @@ class WorkspaceClient:
     """
     The WorkspaceClient is a client for the workspace-level Databricks REST API.
     """
-    def __init__(self, *{{range $args}}, {{.}}: str = None{{end}},
-                 debug_truncate_bytes: int = None,
-                 debug_headers: bool = None,
+    def __init__(self, *{{range $args}}, {{.}}: Optional[str] = None{{end}},
+                 debug_truncate_bytes: Optional[int] = None,
+                 debug_headers: Optional[bool] = None,
                  product="unknown",
                  product_version="0.0.0",
-                 credentials_strategy: CredentialsStrategy = None,
-                 credentials_provider: CredentialsStrategy = None,
-                 config: client.Config = None):
+                 credentials_strategy: Optional[CredentialsStrategy] = None,
+                 credentials_provider: Optional[CredentialsStrategy] = None,
+                 config: Optional[client.Config] = None):
         if not config:
           config = client.Config({{range $args}}{{.}}={{.}}, {{end}}
             credentials_strategy=credentials_strategy,
@@ -110,14 +110,14 @@ class AccountClient:
     The AccountClient is a client for the account-level Databricks REST API.
     """
 
-    def __init__(self, *{{range $args}}, {{.}}: str = None{{end}},
-                 debug_truncate_bytes: int = None,
-                 debug_headers: bool = None,
+    def __init__(self, *{{range $args}}, {{.}}: Optional[str] = None{{end}},
+                 debug_truncate_bytes: Optional[int] = None,
+                 debug_headers: Optional[bool] = None,
                  product="unknown",
                  product_version="0.0.0",
-                 credentials_strategy: CredentialsStrategy = None,
-                 credentials_provider: CredentialsStrategy = None,
-                 config: client.Config = None):
+                 credentials_strategy: Optional[CredentialsStrategy] = None,
+                 credentials_provider: Optional[CredentialsStrategy] = None,
+                 config: Optional[client.Config] = None):
         if not config:
           config = client.Config({{range $args}}{{.}}={{.}}, {{end}}
             credentials_strategy=credentials_strategy,

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Version changelog
 
+## [Release] Release v0.32.1
+
+### Bug Fixes
+
+ * Properly include message when handing SCIM errors ([#753](https://github.com/databricks/databricks-sdk-py/pull/753)).
+
+
+
 ## [Release] Release v0.32.0
 
 ### Bug Fixes

databricks/sdk/__init__.py

@@ -92,11 +92,11 @@ class Config:
     def __init__(self,
                  *,
                  # Deprecated. Use credentials_strategy instead.
-                 credentials_provider: CredentialsStrategy = None,
-                 credentials_strategy: CredentialsStrategy = None,
+                 credentials_provider: Optional[CredentialsStrategy] = None,
+                 credentials_strategy: Optional[CredentialsStrategy] = None,
                  product=None,
                  product_version=None,
-                 clock: Clock = None,
+                 clock: Optional[Clock] = None,
                  **kwargs):
         self._header_factory = None
         self._inner = {}

databricks/sdk/config.py

@@ -10,7 +10,7 @@
 from .config import *
 # To preserve backwards compatibility (as these definitions were previously in this module)
 from .credentials_provider import *
-from .errors import DatabricksError, get_api_error
+from .errors import DatabricksError, _ErrorCustomizer, _Parser
 from .logger import RoundTrip
 from .oauth import retrieve_token
 from .retries import retried
@@ -71,6 +71,8 @@ def __init__(self, cfg: Config = None):
         # Default to 60 seconds
         self._http_timeout_seconds = cfg.http_timeout_seconds if cfg.http_timeout_seconds else 60
 
+        self._error_parser = _Parser(extra_error_customizers=[_AddDebugErrorCustomizer(cfg)])
+
     @property
     def account_id(self) -> str:
         return self._cfg.account_id
@@ -219,27 +221,6 @@ def _is_retryable(err: BaseException) -> Optional[str]:
                 return f'matched {substring}'
         return None
 
-    @classmethod
-    def _parse_retry_after(cls, response: requests.Response) -> Optional[int]:
-        retry_after = response.headers.get("Retry-After")
-        if retry_after is None:
-            # 429 requests should include a `Retry-After` header, but if it's missing,
-            # we default to 1 second.
-            return cls._RETRY_AFTER_DEFAULT
-        # If the request is throttled, try parse the `Retry-After` header and sleep
-        # for the specified number of seconds. Note that this header can contain either
-        # an integer or a RFC1123 datetime string.
-        # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-        #
-        # For simplicity, we only try to parse it as an integer, as this is what Databricks
-        # platform returns. Otherwise, we fall back and don't sleep.
-        try:
-            return int(retry_after)
-        except ValueError:
-            logger.debug(f'Invalid Retry-After header received: {retry_after}. Defaulting to 1')
-            # defaulting to 1 sleep second to make self._is_retryable() simpler
-            return cls._RETRY_AFTER_DEFAULT
-
     def _perform(self,
                  method: str,
                  url: str,
@@ -261,15 +242,8 @@ def _perform(self,
                                          stream=raw,
                                          timeout=self._http_timeout_seconds)
         self._record_request_log(response, raw=raw or data is not None or files is not None)
-        error = get_api_error(response)
+        error = self._error_parser.get_api_error(response)
         if error is not None:
-            status_code = response.status_code
-            is_http_unauthorized_or_forbidden = status_code in (401, 403)
-            is_too_many_requests_or_unavailable = status_code in (429, 503)
-            if is_http_unauthorized_or_forbidden:
-                error.message = self._cfg.wrap_debug_info(error.message)
-            if is_too_many_requests_or_unavailable:
-                error.retry_after_secs = self._parse_retry_after(response)
             raise error from None
         return response
 
@@ -279,6 +253,19 @@ def _record_request_log(self, response: requests.Response, raw: bool = False) ->
         logger.debug(RoundTrip(response, self._cfg.debug_headers, self._debug_truncate_bytes, raw).generate())
 
 
+class _AddDebugErrorCustomizer(_ErrorCustomizer):
+    """An error customizer that adds debug information about the configuration to unauthenticated and
+    unauthorized errors."""
+
+    def __init__(self, cfg: Config):
+        self._cfg = cfg
+
+    def customize_error(self, response: requests.Response, kwargs: dict):
+        if response.status_code in (401, 403):
+            message = kwargs.get('message', 'request failed')
+            kwargs['message'] = self._cfg.wrap_debug_info(message)
+
+
 class StreamingResponse(BinaryIO):
     _response: requests.Response
     _buffer: bytes

databricks/sdk/core.py

@@ -1,6 +1,6 @@
 from .base import DatabricksError, ErrorDetail
-from .mapper import _error_mapper
-from .parser import get_api_error
+from .customizer import _ErrorCustomizer
+from .parser import _Parser
 from .platform import *
 from .private_link import PrivateLinkValidationError
 from .sdk import *

databricks/sdk/errors/__init__.py

@@ -0,0 +1,50 @@
+import abc
+import logging
+
+import requests
+
+
+class _ErrorCustomizer(abc.ABC):
+    """A customizer for errors from the Databricks REST API."""
+
+    @abc.abstractmethod
+    def customize_error(self, response: requests.Response, kwargs: dict):
+        """Customize the error constructor parameters."""
+
+
+class _RetryAfterCustomizer(_ErrorCustomizer):
+    """An error customizer that sets the retry_after_secs parameter based on the Retry-After header."""
+
+    _DEFAULT_RETRY_AFTER_SECONDS = 1
+    """The default number of seconds to wait before retrying a request if the Retry-After header is missing or is not
+    a valid integer."""
+
+    @classmethod
+    def _parse_retry_after(cls, response: requests.Response) -> int:
+        retry_after = response.headers.get("Retry-After")
+        if retry_after is None:
+            logging.debug(
+                f'No Retry-After header received in response with status code 429 or 503. Defaulting to {cls._DEFAULT_RETRY_AFTER_SECONDS}'
+            )
+            # 429 requests should include a `Retry-After` header, but if it's missing,
+            # we default to 1 second.
+            return cls._DEFAULT_RETRY_AFTER_SECONDS
+        # If the request is throttled, try parse the `Retry-After` header and sleep
+        # for the specified number of seconds. Note that this header can contain either
+        # an integer or a RFC1123 datetime string.
+        # See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+        #
+        # For simplicity, we only try to parse it as an integer, as this is what Databricks
+        # platform returns. Otherwise, we fall back and don't sleep.
+        try:
+            return int(retry_after)
+        except ValueError:
+            logging.debug(
+                f'Invalid Retry-After header received: {retry_after}. Defaulting to {cls._DEFAULT_RETRY_AFTER_SECONDS}'
+            )
+            # defaulting to 1 sleep second to make self._is_retryable() simpler
+            return cls._DEFAULT_RETRY_AFTER_SECONDS
+
+    def customize_error(self, response: requests.Response, kwargs: dict):
+        if response.status_code in (429, 503):
+            kwargs['retry_after_secs'] = self._parse_retry_after(response)