feat: improve schema conversion and caching optimizations (#92)

Implement multiple phases of performance improvements, including schema conversion caching, parameter extraction optimizations, and LRU caching for type converters. These changes collectively enhance execution speed and reduce memory usage across various database adapters. All tests pass successfully.

Author: cofin

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
       - id: mixed-line-ending
       - id: trailing-whitespace
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.13.2"
+    rev: "v0.13.3"
     hooks:
       - id: ruff
         args: ["--fix"]

sqlspec/adapters/adbc/type_converter.py

@@ -5,32 +5,65 @@
 MySQL, BigQuery, Snowflake).
 """
 
-from typing import Any
+from functools import lru_cache
+from typing import Any, Final
 
 from sqlspec.core.type_conversion import BaseTypeConverter
 from sqlspec.utils.serializers import to_json
 
+ADBC_SPECIAL_CHARS: Final[frozenset[str]] = frozenset({"{", "[", "-", ":", "T", "."})
+
 
 class ADBCTypeConverter(BaseTypeConverter):
     """ADBC-specific type converter with dialect awareness.
 
     Extends the base BaseTypeConverter with ADBC multi-backend functionality
     including dialect-specific type handling for different database systems.
+    Includes per-instance LRU cache for improved performance.
     """
 
-    __slots__ = ("dialect",)
+    __slots__ = ("_convert_cache", "dialect")
 
-    def __init__(self, dialect: str) -> None:
-        """Initialize with dialect-specific configuration.
+    def __init__(self, dialect: str, cache_size: int = 5000) -> None:
+        """Initialize with dialect-specific configuration and conversion cache.
 
         Args:
             dialect: Target database dialect (postgres, sqlite, duckdb, etc.)
+            cache_size: Maximum number of string values to cache (default: 5000)
         """
         super().__init__()
         self.dialect = dialect.lower()
 
+        @lru_cache(maxsize=cache_size)
+        def _cached_convert(value: str) -> Any:
+            if not value or not any(c in value for c in ADBC_SPECIAL_CHARS):
+                return value
+            detected_type = self.detect_type(value)
+            if detected_type:
+                try:
+                    if self.dialect in {"postgres", "postgresql"}:
+                        if detected_type in {"uuid", "interval"}:
+                            return self.convert_value(value, detected_type)
+                    elif self.dialect == "duckdb":
+                        if detected_type == "uuid":
+                            return self.convert_value(value, detected_type)
+                    elif self.dialect == "sqlite":
+                        if detected_type == "uuid":
+                            return str(value)
+                    elif self.dialect == "bigquery":
+                        if detected_type == "uuid":
+                            return self.convert_value(value, detected_type)
+                    elif self.dialect in {"mysql", "snowflake"} and detected_type in {"uuid", "json"}:
+                        return self.convert_value(value, detected_type)
+                    return self.convert_value(value, detected_type)
+                except Exception:
+                    return value
+            return value
+
+        self._convert_cache = _cached_convert
+
     def convert_if_detected(self, value: Any) -> Any:
-        """Convert value with dialect-specific handling.
+        """Convert value with dialect-specific handling (cached).
 
         Args:
             value: Value to potentially convert.
@@ -40,37 +73,7 @@ def convert_if_detected(self, value: Any) -> Any:
         """
         if not isinstance(value, str):
             return value
-
-        if not any(c in value for c in ["{", "[", "-", ":", "T"]):
-            return value
-
-        detected_type = self.detect_type(value)
-        if detected_type:
-            try:
-                if self.dialect in {"postgres", "postgresql"}:
-                    if detected_type in {"uuid", "interval"}:
-                        return self.convert_value(value, detected_type)
-
-                elif self.dialect == "duckdb":
-                    if detected_type == "uuid":
-                        return self.convert_value(value, detected_type)
-
-                elif self.dialect == "sqlite":
-                    if detected_type == "uuid":
-                        return str(value)
-
-                elif self.dialect == "bigquery":
-                    if detected_type == "uuid":
-                        return self.convert_value(value, detected_type)
-
-                elif self.dialect in {"mysql", "snowflake"} and detected_type in {"uuid", "json"}:
-                    return self.convert_value(value, detected_type)
-
-                return self.convert_value(value, detected_type)
-            except Exception:
-                return value
-
-        return value
+        return self._convert_cache(value)
 
     def convert_dict(self, value: dict[str, Any]) -> Any:
         """Convert dictionary values with dialect-specific handling.
@@ -81,13 +84,8 @@ def convert_dict(self, value: dict[str, Any]) -> Any:
         Returns:
             Converted value appropriate for the dialect.
         """
-
-        # For dialects that cannot handle raw dicts (like ADBC PostgreSQL),
-        # convert to JSON strings
         if self.dialect in {"postgres", "postgresql", "bigquery"}:
             return to_json(value)
-
-        # For other dialects, pass through unchanged
         return value
 
     def supports_native_type(self, type_name: str) -> bool:
@@ -104,11 +102,10 @@ def supports_native_type(self, type_name: str) -> bool:
             "postgresql": ["uuid", "json", "interval", "pg_array"],
             "duckdb": ["uuid", "json"],
             "bigquery": ["json"],
-            "sqlite": [],  # Limited native type support
+            "sqlite": [],
             "mysql": ["json"],
             "snowflake": ["json"],
         }
-
         return type_name in native_support.get(self.dialect, [])
 
     def get_dialect_specific_converter(self, value: Any, target_type: str) -> Any:
@@ -124,36 +121,33 @@ def get_dialect_specific_converter(self, value: Any, target_type: str) -> Any:
         if self.dialect in {"postgres", "postgresql"}:
             if target_type in {"uuid", "json", "interval"}:
                 return self.convert_value(value, target_type)
-
         elif self.dialect == "duckdb":
             if target_type in {"uuid", "json"}:
                 return self.convert_value(value, target_type)
-
         elif self.dialect == "sqlite":
             if target_type == "uuid":
                 return str(value)
             if target_type == "json":
                 return self.convert_value(value, target_type)
-
         elif self.dialect == "bigquery":
             if target_type == "uuid":
                 return str(self.convert_value(value, target_type))
             if target_type == "json":
                 return self.convert_value(value, target_type)
-
         return self.convert_value(value, target_type) if hasattr(self, "convert_value") else value
 
 
-def get_adbc_type_converter(dialect: str) -> ADBCTypeConverter:
+def get_adbc_type_converter(dialect: str, cache_size: int = 5000) -> ADBCTypeConverter:
     """Factory function to create dialect-specific ADBC type converter.
 
     Args:
         dialect: Database dialect name.
+        cache_size: Maximum number of string values to cache (default: 5000)
 
     Returns:
         Configured ADBCTypeConverter instance.
     """
-    return ADBCTypeConverter(dialect)
+    return ADBCTypeConverter(dialect, cache_size)
 
 
-__all__ = ("ADBCTypeConverter", "get_adbc_type_converter")
+__all__ = ("ADBC_SPECIAL_CHARS", "ADBCTypeConverter", "get_adbc_type_converter")

sqlspec/adapters/asyncpg/driver.py

@@ -31,14 +31,60 @@
 logger = get_logger("adapters.asyncpg")
 
 
+def _convert_datetime_param(value: Any) -> Any:
+    """Convert datetime parameter, handling ISO strings.
+
+    Args:
+        value: datetime object or ISO format string
+
+    Returns:
+        datetime object for asyncpg
+    """
+    if isinstance(value, str):
+        return datetime.datetime.fromisoformat(value)
+    return value
+
+
+def _convert_date_param(value: Any) -> Any:
+    """Convert date parameter, handling ISO strings.
+
+    Args:
+        value: date object or ISO format string
+
+    Returns:
+        date object for asyncpg
+    """
+    if isinstance(value, str):
+        return datetime.date.fromisoformat(value)
+    return value
+
+
+def _convert_time_param(value: Any) -> Any:
+    """Convert time parameter, handling ISO strings.
+
+    Args:
+        value: time object or ISO format string
+
+    Returns:
+        time object for asyncpg
+    """
+    if isinstance(value, str):
+        return datetime.time.fromisoformat(value)
+    return value
+
+
 asyncpg_statement_config = StatementConfig(
     dialect="postgres",
     parameter_config=ParameterStyleConfig(
         default_parameter_style=ParameterStyle.NUMERIC,
         supported_parameter_styles={ParameterStyle.NUMERIC, ParameterStyle.POSITIONAL_PYFORMAT},
         default_execution_parameter_style=ParameterStyle.NUMERIC,
         supported_execution_parameter_styles={ParameterStyle.NUMERIC},
-        type_coercion_map={datetime.datetime: lambda x: x, datetime.date: lambda x: x, datetime.time: lambda x: x},
+        type_coercion_map={
+            datetime.datetime: _convert_datetime_param,
+            datetime.date: _convert_date_param,
+            datetime.time: _convert_time_param,
+        },
         has_native_list_expansion=True,
         needs_static_script_compilation=False,
         preserve_parameter_format=True,

sqlspec/adapters/bigquery/type_converter.py

@@ -4,6 +4,7 @@
 for the native BigQuery driver.
 """
 
+from functools import lru_cache
 from typing import Any, Final, Optional
 from uuid import UUID
 
@@ -14,7 +15,6 @@
 except ImportError:
     ScalarQueryParameter = None  # type: ignore[assignment,misc]
 
-# Enhanced BigQuery type mapping with UUID support
 BQ_TYPE_MAP: Final[dict[str, str]] = {
     "str": "STRING",
     "int": "INT64",
@@ -23,23 +23,61 @@
     "datetime": "DATETIME",
     "date": "DATE",
     "time": "TIME",
-    "UUID": "STRING",  # UUID as STRING in BigQuery
+    "UUID": "STRING",
     "uuid": "STRING",
     "Decimal": "NUMERIC",
     "bytes": "BYTES",
     "list": "ARRAY",
     "dict": "STRUCT",
 }
 
+BIGQUERY_SPECIAL_CHARS: Final[frozenset[str]] = frozenset({"{", "[", "-", ":", "T", "."})
+
 
 class BigQueryTypeConverter(BaseTypeConverter):
     """BigQuery-specific type conversion with UUID support.
 
     Extends the base TypeDetector with BigQuery-specific functionality
     including UUID parameter handling for the native BigQuery driver.
+    Includes per-instance LRU cache for improved performance.
     """
 
-    __slots__ = ()
+    __slots__ = ("_convert_cache",)
+
+    def __init__(self, cache_size: int = 5000) -> None:
+        """Initialize converter with per-instance conversion cache.
+
+        Args:
+            cache_size: Maximum number of string values to cache (default: 5000)
+        """
+        super().__init__()
+
+        @lru_cache(maxsize=cache_size)
+        def _cached_convert(value: str) -> Any:
+            if not value or not any(c in value for c in BIGQUERY_SPECIAL_CHARS):
+                return value
+            detected_type = self.detect_type(value)
+            if detected_type:
+                try:
+                    return self.convert_value(value, detected_type)
+                except Exception:
+                    return value
+            return value
+
+        self._convert_cache = _cached_convert
+
+    def convert_if_detected(self, value: Any) -> Any:
+        """Convert string if special type detected (cached).
+
+        Args:
+            value: Value to potentially convert
+
+        Returns:
+            Converted value or original value
+        """
+        if not isinstance(value, str):
+            return value
+        return self._convert_cache(value)
 
     def create_parameter(self, name: str, value: Any) -> Optional[Any]:
         """Create BigQuery parameter with proper type mapping.
@@ -63,7 +101,6 @@ def create_parameter(self, name: str, value: Any) -> Optional[Any]:
                 uuid_obj = convert_uuid(value)
                 return ScalarQueryParameter(name, "STRING", str(uuid_obj))
 
-        # Handle other types
         param_type = BQ_TYPE_MAP.get(type(value).__name__, "STRING")
         return ScalarQueryParameter(name, param_type, value)
 
@@ -78,16 +115,8 @@ def convert_bigquery_value(self, value: Any, column_type: str) -> Any:
             Converted value appropriate for the column type.
         """
         if column_type == "STRING" and isinstance(value, str):
-            # Try to detect if this is a special type
-            detected_type = self.detect_type(value)
-            if detected_type:
-                try:
-                    return self.convert_value(value, detected_type)
-                except Exception:
-                    # If conversion fails, return original value
-                    return value
-
+            return self.convert_if_detected(value)
         return value
 
 
-__all__ = ("BQ_TYPE_MAP", "BigQueryTypeConverter")
+__all__ = ("BIGQUERY_SPECIAL_CHARS", "BQ_TYPE_MAP", "BigQueryTypeConverter")