Update cython to optimize for larger objects

Author: LeanderCS

docs/source/changelog.rst

@@ -3,6 +3,17 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+[0.6.4] - 2025-08-12
+--------------------
+
+Changed
+^^^^^^^
+- Increased freelist sizes for ``InputFilter``, ``FieldModel``, and ``ExternalApiConfig`` to reduce allocations
+- Replaced Python generators with C-style loops and added early exit conditions in filter/validator chains
+- Optimized dictionary operations and string lookups with extended interning
+- Improved ``has_unknown_fields`` with adaptive strategies based on data size
+
+
 [0.6.3] - 2025-08-11
 --------------------
 

flask_inputfilter/_input_filter.pyx

@@ -1,4 +1,8 @@
 # cython: language=c++
+# cython: freelist=256
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: cdivision=True
 
 import json
 import logging
@@ -31,6 +35,12 @@ cdef dict _INTERNED_STRINGS = {
     "required": sys.intern("required"),
     "steps": sys.intern("steps"),
     "validators": sys.intern("validators"),
+    "data": sys.intern("data"),
+    "errors": sys.intern("errors"),
+    "validated_data": sys.intern("validated_data"),
+    "fields": sys.intern("fields"),
+    "name": sys.intern("name"),
+    "value": sys.intern("value"),
 }
 
 cdef extern from "helper.h":
@@ -121,9 +131,18 @@ cdef class InputFilter:
                     Union[Response, tuple[Any, dict[str, Any]]]: The response from the route function or an error response.
                 """
 
-                cdef InputFilter input_filter = cls()
-                cdef string request_method = request.method.encode()
-                if not any(request_method == method for method in input_filter.methods):
+                cdef:
+                    InputFilter input_filter = cls()
+                    string request_method = request.method.encode()
+                    Py_ssize_t i, n = input_filter.methods.size()
+                    bint method_allowed = False
+
+                for i in range(n):
+                    if request_method == input_filter.methods[i]:
+                        method_allowed = True
+                        break
+                
+                if not method_allowed:
                     return Response(status=405, response="Method Not Allowed")
 
                 if request.is_json:
@@ -310,11 +329,13 @@ cdef class InputFilter:
             dict result = {}
             list field_names = list(self.fields.keys())
             str field
+            object value
 
         for i in range(n):
             field = field_names[i]
-            if field in self.data:
-                result[field] = self.data[field]
+            value = self.data.get(field)
+            if value is not None:
+                result[field] = value
         return result
 
     cpdef dict[str, Any] get_unfiltered_data(self):
@@ -347,7 +368,7 @@ cdef class InputFilter:
         """
         self.data = data
 
-    cpdef bint has_unknown(self):
+    cpdef inline bint has_unknown(self):
         """
         Checks whether any values in the current data do not have
         corresponding configurations in the defined fields.

flask_inputfilter/mixins/data_mixin/_data_mixin.pyx

@@ -1,4 +1,7 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: cdivision=True
 
 from typing import Any
 
@@ -10,11 +13,13 @@ from flask_inputfilter.models.cimports cimport BaseFilter, BaseValidator, FieldM
 
 # Compile-time constants for performance thresholds
 DEF LARGE_DATASET_THRESHOLD = 10
+DEF SMALL_DICT_THRESHOLD = 5
 
 
 cdef class DataMixin:
 
     @staticmethod
+    @cython.inline
     cdef bint has_unknown_fields(
             dict[str, Any] data,
             dict[str, FieldModel] fields
@@ -35,19 +40,22 @@ cdef class DataMixin:
         if not data and fields:
             return True
 
-        cdef set field_set
+        cdef:
+            set field_set
+            Py_ssize_t field_count = len(fields)
+            Py_ssize_t data_count = len(data)
 
-        # Use set operations for faster lookup when there are many fields
-        if len(fields) > LARGE_DATASET_THRESHOLD:
+        if field_count > LARGE_DATASET_THRESHOLD and data_count > SMALL_DICT_THRESHOLD:
             field_set = set(fields.keys())
             for field_name in data.keys():
                 if field_name not in field_set:
                     return True
-        else:
-            # Use direct dict lookup for smaller field counts
+        elif data_count < field_count:
             for field_name in data.keys():
                 if field_name not in fields:
                     return True
+        else:
+            return bool(set(data.keys()) - set(fields.keys()))
 
         return False
 
@@ -72,19 +80,30 @@ cdef class DataMixin:
         """
         cdef:
             dict[str, Any] filtered_data = {}
-            Py_ssize_t i, n = len(data) if data else 0
-            list keys = list(data.keys()) if n > 0 else []
-            list values = list(data.values()) if n > 0 else []
+            Py_ssize_t i, n = len(data) if data is not None else 0
+            list keys
+            list values
             str field_name
             object field_value
+            FieldModel field_info
+        
+        # Early return for empty data
+        if n == 0:
+            return filtered_data
+        
+        # Pre-allocate lists
+        keys = list(data.keys())
+        values = list(data.values())
 
         for i in range(n):
             field_name = keys[i]
             field_value = values[i]
 
-            if field_name in fields:
+            # Cache field lookup
+            field_info = fields.get(field_name)
+            if field_info is not None:
                 field_value = ValidationMixin.apply_filters(
-                    fields[field_name].filters,
+                    field_info.filters,
                     global_filters,
                     field_value,
                 )
@@ -94,6 +113,7 @@ cdef class DataMixin:
         return filtered_data
 
     @staticmethod
+    @cython.inline
     cdef tuple validate_with_conditions(
             dict[str, FieldModel] fields,
             dict[str, Any] data,
@@ -125,8 +145,8 @@ cdef class DataMixin:
             fields, data, global_filters, global_validators
         )
 
-        # Check conditions if present and no errors yet
-        if conditions and not errors:
+        # Check conditions only if present and no errors yet
+        if conditions is not None and len(conditions) > 0 and not errors:
             try:
                 ValidationMixin.check_conditions(conditions, validated_data)
             except ValidationError as e:
@@ -150,13 +170,19 @@ cdef class DataMixin:
         cdef:
             Py_ssize_t i, n
             dict source_inputs = source_filter.get_inputs()
-            list keys = list(source_inputs.keys()) if source_inputs else []
-            list new_fields = list(source_inputs.values()) if source_inputs else []
-
-        # Merge fields efficiently
-        n = len(keys)
-        for i in range(n):
-            target_filter.fields[keys[i]] = new_fields[i]
+            list keys
+            list new_fields
+        
+        if source_inputs:
+            keys = list(source_inputs.keys())
+            new_fields = list(source_inputs.values())
+            n = len(keys)
+            
+            # Batch update fields
+            for i in range(n):
+                target_filter.fields[keys[i]] = new_fields[i]
+        else:
+            n = 0
 
         # Merge conditions
         target_filter.conditions.extend(source_filter.conditions)
@@ -174,6 +200,7 @@ cdef class DataMixin:
         )
 
     @staticmethod
+    @cython.inline
     cdef void _merge_component_list(list target_list, list source_list):
         """
         Helper method to merge component lists avoiding duplicates by type.
@@ -183,13 +210,21 @@ cdef class DataMixin:
         - **target_list** (*list*): The list to merge into.
         - **source_list** (*list*): The list to merge from.
         """
-        cdef dict existing_type_map
+        cdef:
+            dict existing_type_map = {}
+            Py_ssize_t i, n = len(target_list)
+            object component
+            type component_type
+        
+        # Build type map once
+        for i in range(n):
+            existing_type_map[type(target_list[i])] = i
 
+        # Process source components
         for component in source_list:
-            existing_type_map = {
-                type(v): i for i, v in enumerate(target_list)
-            }
-            if type(component) in existing_type_map:
-                target_list[existing_type_map[type(component)]] = component
+            component_type = type(component)
+            if component_type in existing_type_map:
+                target_list[existing_type_map[component_type]] = component
             else:
-                target_list.append(component) 
+                target_list.append(component)
+                existing_type_map[component_type] = len(target_list) - 1 

flask_inputfilter/mixins/external_api_mixin/_external_api_mixin.pyx

@@ -1,4 +1,6 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
 
 import re
 from typing import Any
@@ -112,7 +114,8 @@ cdef class ExternalApiMixin:
         return result.get(data_key) if data_key else result
 
     @staticmethod
-    cdef inline str replace_placeholders(
+    @cython.inline
+    cdef str replace_placeholders(
             str value,
             dict[str, Any] validated_data
     ):