fix: correctly handle NULL types for execute_many in the latest ADBC

Author: cofin

sqlspec/adapters/adbc/config.py

@@ -6,6 +6,7 @@
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
 
 from sqlspec.adapters.adbc.driver import AdbcConnection, AdbcDriver
+from sqlspec.adapters.adbc.transformers import AdbcPostgresTransformer
 from sqlspec.config import NoPoolSyncConfig
 from sqlspec.exceptions import ImproperConfigurationError
 from sqlspec.statement.sql import SQLConfig
@@ -434,6 +435,17 @@ def session_manager() -> "Generator[AdbcDriver, None, None]":
                             default_parameter_style=preferred_style,
                         )
 
+                    # Add ADBC PostgreSQL transformer if needed
+                    if self._get_dialect() == "postgres":
+                        # Get the default transformers from the pipeline
+                        pipeline = statement_config.get_statement_pipeline()
+                        existing_transformers = list(pipeline.transformers)
+
+                        # Append our transformer to the existing ones
+                        existing_transformers.append(AdbcPostgresTransformer())
+
+                        statement_config = replace(statement_config, transformers=existing_transformers)
+
                 driver = self.driver_type(connection=connection, config=statement_config)
                 yield driver
 

sqlspec/adapters/adbc/driver.py

@@ -220,6 +220,9 @@ def _execute(
 
             with self._get_cursor(txn_conn) as cursor:
                 try:
+                    # ADBC PostgreSQL has issues with NULL parameters in some cases
+                    # The transformer handles all-NULL cases, but mixed NULL/non-NULL
+                    # can still cause "Can't map Arrow type 'na' to Postgres type" errors
                     cursor.execute(sql, cursor_params or [])
                 except Exception as e:
                     # Rollback transaction on error for PostgreSQL to avoid
@@ -265,6 +268,17 @@ def _execute_many(
             # Normalize parameter list using consolidated utility
             converted_param_list = convert_parameter_sequence(param_list)
 
+            # Handle empty parameter list case for PostgreSQL
+            if not converted_param_list and self.dialect == "postgres":
+                # Return empty result without executing
+                return SQLResult(
+                    statement=SQL(sql, _dialect=self.dialect),
+                    data=[],
+                    rows_affected=0,
+                    operation_type="EXECUTE",
+                    metadata={"status_message": "OK"},
+                )
+
             with self._get_cursor(txn_conn) as cursor:
                 try:
                     cursor.executemany(sql, converted_param_list or [])

sqlspec/adapters/adbc/transformers.py

@@ -0,0 +1,108 @@
+"""ADBC-specific AST transformers for handling driver limitations."""
+
+from typing import Optional
+
+from sqlglot import exp
+
+from sqlspec.protocols import ProcessorProtocol
+from sqlspec.statement.pipelines.context import SQLProcessingContext
+
+__all__ = ("AdbcPostgresTransformer",)
+
+
+class AdbcPostgresTransformer(ProcessorProtocol):
+    """Transformer to handle ADBC PostgreSQL driver limitations.
+
+    This transformer addresses specific issues with the ADBC PostgreSQL driver:
+    1. Empty parameter lists in executemany() causing "no parameter $1" errors
+    2. NULL parameters causing "Can't map Arrow type 'na' to Postgres type" errors
+
+    The transformer works at the AST level to properly handle these edge cases.
+    """
+
+    def __init__(self) -> None:
+        self.has_placeholders = False
+        self.all_params_null = False
+        self.is_empty_params = False
+        self.has_null_params = False
+        self.null_param_indices: list[int] = []
+
+    def process(self, expression: Optional[exp.Expression], context: SQLProcessingContext) -> Optional[exp.Expression]:
+        """Process the SQL expression to handle ADBC limitations."""
+        if not expression:
+            return expression
+
+        # Check if we have an empty parameter list for executemany
+        # Look at the merged_parameters in the context
+        params = context.merged_parameters
+
+        # For execute_many, check if we have an empty list
+        if isinstance(params, list) and len(params) == 0:
+            self.is_empty_params = True
+
+        # Check for NULL parameters
+        if params:
+            if isinstance(params, (list, tuple)):
+                # Track which parameters are NULL
+                self.null_param_indices = [i for i, p in enumerate(params) if p is None]
+                self.has_null_params = len(self.null_param_indices) > 0
+                self.all_params_null = len(self.null_param_indices) == len(params)
+
+                # For ADBC PostgreSQL, we need to replace NULL parameters with literals
+                # and remove them from the parameter list
+                if self.has_null_params:
+                    # Create new parameter list without NULLs
+                    new_params = [p for p in params if p is not None]
+                    context.merged_parameters = new_params
+
+            elif isinstance(params, dict):
+                # For dict parameters, track which ones are NULL
+                null_keys = [k for k, v in params.items() if v is None]
+                self.has_null_params = len(null_keys) > 0
+                self.all_params_null = len(null_keys) == len(params)
+
+                if self.has_null_params:
+                    # Remove NULL parameters from dict
+                    context.merged_parameters = {k: v for k, v in params.items() if v is not None}
+
+        # Transform the AST if needed
+        if self.is_empty_params:
+            # For empty parameters, we should skip transformation and let the driver handle it
+            # The driver already has logic to return empty result for empty params
+            return expression
+
+        if self.has_null_params:
+            # Transform placeholders to NULL literals where needed
+            self._parameter_index = 0  # Track current parameter position
+            return expression.transform(self._transform_node)
+
+        return expression
+
+    def _transform_node(self, node: exp.Expression) -> exp.Expression:
+        """Transform individual AST nodes."""
+        # Handle parameter nodes (e.g., $1, $2, etc. in PostgreSQL)
+        if isinstance(node, exp.Parameter):
+            # Access the parameter value directly from the AST node
+            # The 'this' attribute contains a Literal node, whose 'this' contains the actual value
+            if node.this and isinstance(node.this, exp.Literal):
+                try:
+                    param_index = int(node.this.this) - 1  # Convert to 0-based index
+                    # Check if this parameter should be NULL
+                    if param_index in self.null_param_indices:
+                        return exp.Null()
+                    # Renumber the parameter based on how many NULLs came before it
+                    nulls_before = sum(1 for idx in self.null_param_indices if idx < param_index)
+                    new_index = param_index - nulls_before + 1  # Convert back to 1-based
+                    return exp.Parameter(this=exp.Literal.number(new_index))
+                except (ValueError, IndexError):
+                    pass
+
+        # Handle placeholder nodes for other dialects
+        elif isinstance(node, exp.Placeholder):
+            # For placeholders, we need to track position
+            if self._parameter_index in self.null_param_indices:
+                self._parameter_index += 1
+                return exp.Null()
+            self._parameter_index += 1
+
+        return node

sqlspec/statement/parameters.py

@@ -10,16 +10,14 @@
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from typing import Any, Final, Optional, Union
 
+from sqlglot import exp
 from typing_extensions import TypedDict
 
 from sqlspec.exceptions import ExtraParameterError, MissingParameterError, ParameterStyleMismatchError
 from sqlspec.typing import SQLParameterType
 
-if TYPE_CHECKING:
-    from sqlglot import exp
-
 # Constants
 MAX_32BIT_INT: Final[int] = 2147483647
 
@@ -28,7 +26,7 @@
     "ParameterConverter",
     "ParameterInfo",
     "ParameterStyle",
-    "ParameterStyleTransformationState",
+    "ParameterStyleConversionState",
     "ParameterValidator",
     "SQLParameterType",
     "TypedParameter",
@@ -169,7 +167,7 @@ class ParameterStyleInfo(TypedDict, total=False):
 
 
 @dataclass
-class ParameterStyleTransformationState:
+class ParameterStyleConversionState:
     """Encapsulates all information about parameter style transformation.
 
     This class provides a single source of truth for parameter style conversions,
@@ -213,7 +211,7 @@ class ConvertedParameters:
     merged_parameters: "SQLParameterType"
     """Parameters after merging from various sources."""
 
-    conversion_state: ParameterStyleTransformationState
+    conversion_state: ParameterStyleConversionState
     """Complete conversion state for tracking conversions."""
 
 
@@ -314,17 +312,13 @@ def get_parameter_style(parameters_info: "list[ParameterInfo]") -> "ParameterSty
         """
         if not parameters_info:
             return ParameterStyle.NONE
-
-        # Note: This logic prioritizes pyformat if present, then named, then positional.
         is_pyformat_named = any(p.style == ParameterStyle.NAMED_PYFORMAT for p in parameters_info)
         is_pyformat_positional = any(p.style == ParameterStyle.POSITIONAL_PYFORMAT for p in parameters_info)
 
         if is_pyformat_named:
             return ParameterStyle.NAMED_PYFORMAT
-        if is_pyformat_positional:  # If only PYFORMAT_POSITIONAL and not PYFORMAT_NAMED
+        if is_pyformat_positional:
             return ParameterStyle.POSITIONAL_PYFORMAT
-
-        # Simplified logic if not pyformat, checks for any named or any positional
         has_named = any(
             p.style
             in {
@@ -336,13 +330,7 @@ def get_parameter_style(parameters_info: "list[ParameterInfo]") -> "ParameterSty
             for p in parameters_info
         )
         has_positional = any(p.style in {ParameterStyle.QMARK, ParameterStyle.NUMERIC} for p in parameters_info)
-
-        # If mixed named and positional (non-pyformat), prefer named as dominant.
-        # The choice of NAMED_COLON here is somewhat arbitrary if multiple named styles are mixed.
         if has_named:
-            # Could refine to return the style of the first named param encountered, or most frequent.
-            # For simplicity, returning a general named style like NAMED_COLON is often sufficient.
-            # Or, more accurately, find the first named style:
             for p_style in (
                 ParameterStyle.NAMED_COLON,
                 ParameterStyle.POSITIONAL_COLON,
@@ -354,12 +342,11 @@ def get_parameter_style(parameters_info: "list[ParameterInfo]") -> "ParameterSty
             return ParameterStyle.NAMED_COLON
 
         if has_positional:
-            # Similarly, could choose QMARK or NUMERIC based on presence.
             if any(p.style == ParameterStyle.NUMERIC for p in parameters_info):
                 return ParameterStyle.NUMERIC
-            return ParameterStyle.QMARK  # Default positional
+            return ParameterStyle.QMARK
 
-        return ParameterStyle.NONE  # Should not be reached if parameters_info is not empty
+        return ParameterStyle.NONE
 
     @staticmethod
     def determine_parameter_input_type(parameters_info: "list[ParameterInfo]") -> "Optional[type]":
@@ -384,9 +371,8 @@ def determine_parameter_input_type(parameters_info: "list[ParameterInfo]") -> "O
         if any(
             p.name is not None and p.style not in {ParameterStyle.POSITIONAL_COLON, ParameterStyle.NUMERIC}
             for p in parameters_info
-        ):  # True for NAMED styles and PYFORMAT_NAMED
+        ):
             return dict
-        # All parameters must have p.name is None or be positional styles (POSITIONAL_COLON, NUMERIC)
         if all(
             p.name is None or p.style in {ParameterStyle.POSITIONAL_COLON, ParameterStyle.NUMERIC}
             for p in parameters_info
@@ -400,9 +386,7 @@ def determine_parameter_input_type(parameters_info: "list[ParameterInfo]") -> "O
             "Ambiguous parameter structure for determining input type. "
             "Query might contain a mix of named and unnamed styles not typically supported together."
         )
-        # Defaulting to dict if any named param is found, as that's the more common requirement for mixed scenarios.
-        # However, strict validation should ideally prevent such mixed styles from being valid.
-        return dict  # Or raise an error for unsupported mixed styles.
+        return dict
 
     def validate_parameters(
         self,
@@ -421,12 +405,7 @@ def validate_parameters(
             ParameterStyleMismatchError: When style doesn't match
         """
         expected_input_type = self.determine_parameter_input_type(parameters_info)
-
-        # Allow creating SQL statements with placeholders but no parameters
-        # This enables patterns like SQL("SELECT * FROM users WHERE id = ?").as_many([...])
-        # Validation will happen later when parameters are actually provided
         if provided_params is None and parameters_info:
-            # Don't raise an error, just return - validation will happen later
             return
 
         if (
@@ -707,7 +686,7 @@ def convert_parameters(
             self.validator.validate_parameters(parameters_info, merged_params, sql)
         if needs_conversion:
             transformed_sql, placeholder_map = self._transform_sql_for_parsing(sql, parameters_info)
-            conversion_state = ParameterStyleTransformationState(
+            conversion_state = ParameterStyleConversionState(
                 was_transformed=True,
                 original_styles=list({p.style for p in parameters_info}),
                 transformation_style=ParameterStyle.NAMED_COLON,
@@ -716,7 +695,7 @@ def convert_parameters(
             )
         else:
             transformed_sql = sql
-            conversion_state = ParameterStyleTransformationState(
+            conversion_state = ParameterStyleConversionState(
                 was_transformed=False,
                 original_styles=list({p.style for p in parameters_info}),
                 original_param_info=parameters_info,
@@ -775,10 +754,10 @@ def merge_parameters(
             return parameters
 
         if kwargs is not None:
-            return dict(kwargs)  # Make a copy
+            return dict(kwargs)
 
         if args is not None:
-            return list(args)  # Convert tuple of args to list for consistency and mutability if needed later
+            return list(args)
 
         return None
 
@@ -809,53 +788,34 @@ def wrap_parameters_with_types(
 
         def infer_type_from_value(value: Any) -> tuple[str, "exp.DataType"]:
             """Infer SQL type hint and SQLGlot DataType from Python value."""
-            # Import here to avoid issues
-            from sqlglot import exp
 
             # None/NULL
             if value is None:
                 return "null", exp.DataType.build("NULL")
-
-            # Boolean
             if isinstance(value, bool):
                 return "boolean", exp.DataType.build("BOOLEAN")
-
-            # Integer types
             if isinstance(value, int) and not isinstance(value, bool):
                 if abs(value) > MAX_32BIT_INT:
                     return "bigint", exp.DataType.build("BIGINT")
                 return "integer", exp.DataType.build("INT")
-
-            # Float/Decimal
             if isinstance(value, float):
                 return "float", exp.DataType.build("FLOAT")
             if isinstance(value, Decimal):
                 return "decimal", exp.DataType.build("DECIMAL")
-
-            # Date/Time types
             if isinstance(value, datetime):
                 return "timestamp", exp.DataType.build("TIMESTAMP")
             if isinstance(value, date):
                 return "date", exp.DataType.build("DATE")
             if isinstance(value, time):
                 return "time", exp.DataType.build("TIME")
-
-            # JSON/Dict
             if isinstance(value, dict):
                 return "json", exp.DataType.build("JSON")
-
-            # Array/List
             if isinstance(value, (list, tuple)):
                 return "array", exp.DataType.build("ARRAY")
-
             if isinstance(value, str):
                 return "string", exp.DataType.build("VARCHAR")
-
-            # Bytes
             if isinstance(value, bytes):
                 return "binary", exp.DataType.build("BINARY")
-
-            # Default fallback
             return "string", exp.DataType.build("VARCHAR")
 
         def wrap_value(value: Any, semantic_name: Optional[str] = None) -> Any: