Merge branch 'main' into jprakash-db/complex-param

Merged Main

Author: jprakash-db

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release History
 
+# 4.0.3 (2025-04-22)
+
+- Fix: Removed `packaging` dependency in favour of default libraries, for `urllib3` version checks (databricks/databricks-sql-python#547 by @jprakash-db)
+- 
 # 4.0.2 (2025-04-01)
 
 - Fix: Relaxed the pin for `python-dateutil` to be `^2.8.0` (databricks/databricks-sql-python#538 by @jprakash-db)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databricks-sql-connector"
-version = "4.0.2"
+version = "4.0.3"
 description = "Databricks SQL Connector for Python"
 authors = ["Databricks <[email protected]>"]
 license = "Apache-2.0"

src/databricks/sql/__init__.py

@@ -68,7 +68,7 @@ def __repr__(self):
 DATE = DBAPITypeObject("date")
 ROWID = DBAPITypeObject()
 
-__version__ = "4.0.2"
+__version__ = "4.0.3"
 USER_AGENT_NAME = "PyDatabricksSqlConnector"
 
 # These two functions are pyhive legacy

src/databricks/sql/auth/retry.py

@@ -2,6 +2,7 @@
 import random
 import time
 import typing
+from importlib.metadata import version
 from enum import Enum
 from typing import List, Optional, Tuple, Union
 
@@ -16,7 +17,6 @@
     from urllib3 import HTTPResponse as BaseHTTPResponse
 from urllib3 import Retry
 from urllib3.util.retry import RequestHistory
-from packaging import version
 
 
 from databricks.sql.exc import (
@@ -312,7 +312,9 @@ def get_backoff_time(self) -> float:
 
         current_attempt = self.stop_after_attempts_count - int(self.total or 0)
         proposed_backoff = (2**current_attempt) * self.delay_min
-        if version.parse(urllib3.__version__) >= version.parse("2.0.0"):
+
+        library_version = version("urllib3")
+        if int(library_version.split(".")[0]) >= 2:
             if self.backoff_jitter != 0.0:
                 proposed_backoff += random.random() * self.backoff_jitter
 

src/databricks/sql/client.py

@@ -1417,9 +1417,22 @@ def fetchall_arrow(self) -> "pyarrow.Table":
         while not self.has_been_closed_server_side and self.has_more_rows:
             self._fill_results_buffer()
             partial_results = self.results.remaining_rows()
-            results = pyarrow.concat_tables([results, partial_results])
+            if isinstance(results, ColumnTable) and isinstance(
+                partial_results, ColumnTable
+            ):
+                results = self.merge_columnar(results, partial_results)
+            else:
+                results = pyarrow.concat_tables([results, partial_results])
             self._next_row_index += partial_results.num_rows
 
+        # If PyArrow is installed and we have a ColumnTable result, convert it to PyArrow Table
+        # Valid only for metadata commands result set
+        if isinstance(results, ColumnTable) and pyarrow:
+            data = {
+                name: col
+                for name, col in zip(results.column_names, results.column_table)
+            }
+            return pyarrow.Table.from_pydict(data)
         return results
 
     def fetchall_columnar(self):