Bump python sdk version (#827)

Author: eric-wang-1990

CHANGELOG.md

@@ -2,10 +2,14 @@
 
 ### Features
 
+- Support databricks OAuth M2M auth type. Updated OAuth readme doc with instructions.([827](https://github.com/databricks/dbt-databricks/pull/827))
+
 - Introduced use_materialization_v2 flag for gating materialization revamps. ([844](https://github.com/databricks/dbt-databricks/pull/844))
 
 ### Under the Hood
 
+- Update pinned python SDK version from 0.17.0 to 0.41.0. ([827](https://github.com/databricks/dbt-databricks/pull/827))
+
 - Implement new constraint logic for use_materialization_v2 flag ([846](https://github.com/databricks/dbt-databricks/pull/846/files)), ([876](https://github.com/databricks/dbt-databricks/pull/876))
 
 ## dbt-databricks 1.9.5 (TBD)

dbt/adapters/databricks/api_client.py

@@ -13,8 +13,7 @@
 
 from dbt.adapters.databricks import utils
 from dbt.adapters.databricks.__version__ import version
-from dbt.adapters.databricks.auth import BearerAuth
-from dbt.adapters.databricks.credentials import DatabricksCredentials
+from dbt.adapters.databricks.credentials import BearerAuth, DatabricksCredentials
 from dbt.adapters.databricks.logging import logger
 
 DEFAULT_POLLING_INTERVAL = 10
@@ -557,8 +556,7 @@ def create(
         http_headers = credentials.get_all_http_headers(
             connection_parameters.pop("http_headers", {})
         )
-        credentials_provider = credentials.authenticate(None)
-        header_factory = credentials_provider(None)  # type: ignore
+        header_factory = credentials.authenticate().credentials_provider()
         session.auth = BearerAuth(header_factory)
 
         session.headers.update({"User-Agent": user_agent, **http_headers})

dbt/adapters/databricks/auth.py

@@ -32,7 +32,10 @@
 )
 from dbt.adapters.databricks.__version__ import version as __version__
 from dbt.adapters.databricks.api_client import DatabricksApiClient
-from dbt.adapters.databricks.credentials import DatabricksCredentials, TCredentialProvider
+from dbt.adapters.databricks.credentials import (
+    DatabricksCredentialManager,
+    DatabricksCredentials,
+)
 from dbt.adapters.databricks.events.connection_events import (
     ConnectionAcquire,
     ConnectionCancel,
@@ -373,7 +376,7 @@ def _reset_handle(self, open: Callable[[Connection], Connection]) -> None:
 
 class DatabricksConnectionManager(SparkConnectionManager):
     TYPE: str = "databricks"
-    credentials_provider: Optional[TCredentialProvider] = None
+    credentials_manager: Optional[DatabricksCredentialManager] = None
     _user_agent = f"dbt-databricks/{__version__}"
 
     def __init__(self, profile: AdapterRequiredConfig, mp_context: SpawnContext):
@@ -634,7 +637,7 @@ def open(cls, connection: Connection) -> Connection:
         timeout = creds.connect_timeout
 
         # gotta keep this so we don't prompt users many times
-        cls.credentials_provider = creds.authenticate(cls.credentials_provider)
+        cls.credentials_manager = creds.authenticate()
 
         invocation_env = creds.get_invocation_env()
         user_agent_entry = cls._user_agent
@@ -652,12 +655,13 @@ def open(cls, connection: Connection) -> Connection:
         http_path = databricks_connection.http_path
 
         def connect() -> DatabricksSQLConnectionWrapper:
+            assert cls.credentials_manager is not None
             try:
                 # TODO: what is the error when a user specifies a catalog they don't have access to
                 conn = dbsql.connect(
                     server_hostname=creds.host,
                     http_path=http_path,
-                    credentials_provider=cls.credentials_provider,
+                    credentials_provider=cls.credentials_manager.credentials_provider,
                     http_headers=http_headers if http_headers else None,
                     session_configuration=creds.session_properties,
                     catalog=creds.database,
@@ -708,7 +712,7 @@ def _open(cls, connection: Connection, query_header_context: Any = None) -> Conn
         timeout = creds.connect_timeout
 
         # gotta keep this so we don't prompt users many times
-        cls.credentials_provider = creds.authenticate(cls.credentials_provider)
+        cls.credentials_manager = creds.authenticate()
 
         invocation_env = creds.get_invocation_env()
         user_agent_entry = cls._user_agent

dbt/adapters/databricks/connections.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Any, ClassVar, Optional, Type, TypeVar
+from typing import Any, ClassVar, Optional, TypeVar
 
 from dbt_common.contracts.constraints import (
     ColumnLevelConstraint,
@@ -32,7 +32,7 @@ class TypedConstraint(ModelLevelConstraint, ABC):
     str_type: ClassVar[str]
 
     @classmethod
-    def __post_deserialize__(cls: Type[T], obj: T) -> T:
+    def __post_deserialize__(cls: type[T], obj: T) -> T:
         assert obj.type == cls.str_type, "Mismatched constraint type"
         obj._validate()
         return obj