[Fix] Add support for all error details types and fix potential unmarshalling error (#912)

## What changes are proposed in this pull request?

This PR refactors how `DatabricksError` processes error details to fully
support the Databricks Error specification. In particular, it provides a
clear mechanism to access known error details types while still enabling
users to access unknown error details.

The core of the PR is the addition of a new `ErrorDetails` type which
operates as the union of all error details type. It implements the fact
that an error response should only contain at most one instance of each
type.

The code is purposely structured to hide implementation details by
separating the exported error details types from their unmarshalling
logic. This arguably leads to code that might be more complicated that
it ought to be. We could consider further simplifying it in the future
(e.g. by relying on the corresponding proto unmarshallers).

## How is this tested?

100% unit tests coverage of new code + slight refactor of the test
suite.

Author: renaudhartert-db

NEXT_CHANGELOG.md

@@ -9,6 +9,8 @@
 ### Documentation
 
 ### Internal Changes
+
+* Refactor `DatabricksError` to expose different types of error details ([#912](https://github.com/databricks/databricks-sdk-py/pull/912)). 
 * Update Jobs ListJobs API to support paginated responses ([#896](https://github.com/databricks/databricks-sdk-py/pull/896))
 * Update Jobs ListRuns API to support paginated responses ([#890](https://github.com/databricks/databricks-sdk-py/pull/890))
 * Introduce automated tagging ([#888](https://github.com/databricks/databricks-sdk-py/pull/888))

databricks/sdk/errors/base.py

@@ -5,7 +5,10 @@
 
 import requests
 
+from . import details as errdetails
 
+
+# Deprecated.
 class ErrorDetail:
     def __init__(
         self,
@@ -22,17 +25,18 @@ def __init__(
 
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> "ErrorDetail":
-        if "@type" in d:
-            d["type"] = d["@type"]
-        return cls(**d)
+        # Key "@type" is not a valid keyword argument name in Python. Rename
+        # it to "type" to avoid conflicts.
+        safe_args = {}
+        for k, v in d.items():
+            safe_args[k if k != "@type" else "type"] = v
+
+        return cls(**safe_args)
 
 
 class DatabricksError(IOError):
     """Generic error from Databricks REST API"""
 
-    # Known ErrorDetail types
-    _error_info_type = "type.googleapis.com/google.rpc.ErrorInfo"
-
     def __init__(
         self,
         message: str = None,
@@ -54,11 +58,11 @@ def __init__(
         :param status: [Deprecated]
         :param scimType: [Deprecated]
         :param error: [Deprecated]
-        :param retry_after_secs:
+        :param retry_after_secs: [Deprecated]
         :param details:
         :param kwargs:
         """
-        # SCIM-specific parameters are deprecated
+        # SCIM-specific parameters are deprecated.
         if detail:
             warnings.warn(
                 "The 'detail' parameter of DatabricksError is deprecated and will be removed in a future version."
@@ -72,12 +76,18 @@ def __init__(
                 "The 'status' parameter of DatabricksError is deprecated and will be removed in a future version."
             )
 
-        # API 1.2-specific parameters are deprecated
+        # API 1.2-specific parameters are deprecated.
         if error:
             warnings.warn(
                 "The 'error' parameter of DatabricksError is deprecated and will be removed in a future version."
             )
 
+        # Retry-after is deprecated.
+        if retry_after_secs:
+            warnings.warn(
+                "The 'retry_after_secs' parameter of DatabricksError is deprecated and will be removed in a future version."
+            )
+
         if detail:
             # Handle SCIM error message details
             # @see https://tools.ietf.org/html/rfc7644#section-3.7.3
@@ -88,20 +98,32 @@ def __init__(
             # add more context from SCIM responses
             message = f"{scimType} {message}".strip(" ")
             error_code = f"SCIM_{status}"
+
         super().__init__(message if message else error)
         self.error_code = error_code
         self.retry_after_secs = retry_after_secs
-        self.details = [ErrorDetail.from_dict(detail) for detail in details] if details else []
+        self._error_details = errdetails.parse_error_details(details)
         self.kwargs = kwargs
 
+        # Deprecated.
+        self.details = []
+        if details:
+            for d in details:
+                if not isinstance(d, dict):
+                    continue
+                self.details.append(ErrorDetail.from_dict(d))
+
     def get_error_info(self) -> List[ErrorDetail]:
-        return self._get_details_by_type(DatabricksError._error_info_type)
+        return self._get_details_by_type(errdetails._ERROR_INFO_TYPE)
 
     def _get_details_by_type(self, error_type) -> List[ErrorDetail]:
-        if self.details == None:
+        if self.details is None:
             return []
         return [detail for detail in self.details if detail.type == error_type]
 
+    def get_error_details(self) -> errdetails.ErrorDetails:
+        return self._error_details
+
 
 @dataclass
 class _ErrorOverride: