Update cython to optimize for larger objects

Author: LeanderCS

docs/source/changelog.rst

@@ -3,6 +3,17 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+[0.6.4] - 2025-08-12
+--------------------
+
+Changed
+^^^^^^^
+- Increased freelist sizes for ``InputFilter``, ``FieldModel``, and ``ExternalApiConfig`` to reduce allocations
+- Replaced Python generators with C-style loops and added early exit conditions in filter/validator chains
+- Optimized dictionary operations and string lookups with extended interning
+- Improved ``has_unknown_fields`` with adaptive strategies based on data size
+
+
 [0.6.3] - 2025-08-11
 --------------------
 

flask_inputfilter/_input_filter.pyx

@@ -1,4 +1,8 @@
 # cython: language=c++
+# cython: freelist=256
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: cdivision=True
 
 import json
 import logging
@@ -31,6 +35,12 @@ cdef dict _INTERNED_STRINGS = {
     "required": sys.intern("required"),
     "steps": sys.intern("steps"),
     "validators": sys.intern("validators"),
+    "data": sys.intern("data"),
+    "errors": sys.intern("errors"),
+    "validated_data": sys.intern("validated_data"),
+    "fields": sys.intern("fields"),
+    "name": sys.intern("name"),
+    "value": sys.intern("value"),
 }
 
 cdef extern from "helper.h":
@@ -121,9 +131,18 @@ cdef class InputFilter:
                     Union[Response, tuple[Any, dict[str, Any]]]: The response from the route function or an error response.
                 """
 
-                cdef InputFilter input_filter = cls()
-                cdef string request_method = request.method.encode()
-                if not any(request_method == method for method in input_filter.methods):
+                cdef:
+                    InputFilter input_filter = cls()
+                    string request_method = request.method.encode()
+                    Py_ssize_t i, n = input_filter.methods.size()
+                    bint method_allowed = False
+
+                for i in range(n):
+                    if request_method == input_filter.methods[i]:
+                        method_allowed = True
+                        break
+                
+                if not method_allowed:
                     return Response(status=405, response="Method Not Allowed")
 
                 if request.is_json:
@@ -310,11 +329,13 @@ cdef class InputFilter:
             dict result = {}
             list field_names = list(self.fields.keys())
             str field
+            object value
 
         for i in range(n):
             field = field_names[i]
-            if field in self.data:
-                result[field] = self.data[field]
+            value = self.data.get(field)
+            if value is not None:
+                result[field] = value
         return result
 
     cpdef dict[str, Any] get_unfiltered_data(self):

flask_inputfilter/mixins/data_mixin/_data_mixin.pyx

@@ -1,4 +1,7 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: cdivision=True
 
 from typing import Any
 
@@ -10,6 +13,7 @@ from flask_inputfilter.models.cimports cimport BaseFilter, BaseValidator, FieldM
 
 # Compile-time constants for performance thresholds
 DEF LARGE_DATASET_THRESHOLD = 10
+DEF SMALL_DICT_THRESHOLD = 5
 
 
 cdef class DataMixin:
@@ -35,19 +39,22 @@ cdef class DataMixin:
         if not data and fields:
             return True
 
-        cdef set field_set
+        cdef:
+            set field_set
+            Py_ssize_t field_count = len(fields)
+            Py_ssize_t data_count = len(data)
 
-        # Use set operations for faster lookup when there are many fields
-        if len(fields) > LARGE_DATASET_THRESHOLD:
+        if field_count > LARGE_DATASET_THRESHOLD and data_count > SMALL_DICT_THRESHOLD:
             field_set = set(fields.keys())
             for field_name in data.keys():
                 if field_name not in field_set:
                     return True
-        else:
-            # Use direct dict lookup for smaller field counts
+        elif data_count < field_count:
             for field_name in data.keys():
                 if field_name not in fields:
                     return True
+        else:
+            return bool(set(data.keys()) - set(fields.keys()))
 
         return False
 
@@ -72,19 +79,27 @@ cdef class DataMixin:
         """
         cdef:
             dict[str, Any] filtered_data = {}
-            Py_ssize_t i, n = len(data) if data else 0
-            list keys = list(data.keys()) if n > 0 else []
-            list values = list(data.values()) if n > 0 else []
+            Py_ssize_t i, n = len(data) if data is not None else 0
+            list keys
+            list values
             str field_name
             object field_value
+            FieldModel field_info
+
+        if n == 0:
+            return filtered_data
+
+        keys = list(data.keys())
+        values = list(data.values())
 
         for i in range(n):
             field_name = keys[i]
             field_value = values[i]
-            
-            if field_name in fields:
+
+            field_info = fields.get(field_name)
+            if field_info is not None:
                 field_value = ValidationMixin.apply_filters(
-                    fields[field_name].filters,
+                    field_info.filters,
                     global_filters,
                     field_value,
                 )
@@ -125,8 +140,8 @@ cdef class DataMixin:
             fields, data, global_filters, global_validators
         )
 
-        # Check conditions if present and no errors yet
-        if conditions and not errors:
+        # Check conditions only if present and no errors yet
+        if conditions is not None and len(conditions) > 0 and not errors:
             try:
                 ValidationMixin.check_conditions(conditions, validated_data)
             except ValidationError as e:
@@ -150,13 +165,18 @@ cdef class DataMixin:
         cdef:
             Py_ssize_t i, n
             dict source_inputs = source_filter.get_inputs()
-            list keys = list(source_inputs.keys()) if source_inputs else []
-            list new_fields = list(source_inputs.values()) if source_inputs else []
+            list keys
+            list new_fields
+        
+        if source_inputs:
+            keys = list(source_inputs.keys())
+            new_fields = list(source_inputs.values())
+            n = len(keys)
 
-        # Merge fields efficiently
-        n = len(keys)
-        for i in range(n):
-            target_filter.fields[keys[i]] = new_fields[i]
+            for i in range(n):
+                target_filter.fields[keys[i]] = new_fields[i]
+        else:
+            n = 0
 
         # Merge conditions
         target_filter.conditions.extend(source_filter.conditions)
@@ -183,13 +203,19 @@ cdef class DataMixin:
         - **target_list** (*list*): The list to merge into.
         - **source_list** (*list*): The list to merge from.
         """
-        cdef dict existing_type_map
-        
+        cdef:
+            dict existing_type_map = {}
+            Py_ssize_t i, n = len(target_list)
+            object component
+            type component_type
+
+        for i in range(n):
+            existing_type_map[type(target_list[i])] = i
+
         for component in source_list:
-            existing_type_map = {
-                type(v): i for i, v in enumerate(target_list)
-            }
-            if type(component) in existing_type_map:
-                target_list[existing_type_map[type(component)]] = component
+            component_type = type(component)
+            if component_type in existing_type_map:
+                target_list[existing_type_map[component_type]] = component
             else:
-                target_list.append(component) 
+                target_list.append(component)
+                existing_type_map[component_type] = len(target_list) - 1

flask_inputfilter/mixins/external_api_mixin/_external_api_mixin.pyx

@@ -1,4 +1,6 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
 
 import re
 from typing import Any

flask_inputfilter/mixins/validation_mixin/_validation_mixin.pyx

@@ -1,4 +1,8 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
+# cython: cdivision=True
+# cython: nonecheck=False
 
 import cython
 from typing import Any
@@ -38,18 +42,23 @@ cdef class ValidationMixin:
             return None
 
         cdef:
-            Py_ssize_t i, n
+            Py_ssize_t i, n1, n2
             BaseFilter current_filter
 
-        n = len(filters1) if filters1 else 0
-        for i in range(n):
+        n1 = len(filters1) if filters1 is not None else 0
+        n2 = len(filters2) if filters2 is not None else 0
+
+        for i in range(n1):
             current_filter = filters1[i]
             value = current_filter.apply(value)
+            if value is None:
+                return None
 
-        n = len(filters2) if filters2 else 0
-        for i in range(n):
+        for i in range(n2):
             current_filter = filters2[i]
             value = current_filter.apply(value)
+            if value is None:
+                return None
 
         return value
 
@@ -92,16 +101,16 @@ cdef class ValidationMixin:
             return None
 
         cdef:
-            Py_ssize_t i, n = len(steps) if steps else 0
+            Py_ssize_t i, n = len(steps) if steps is not None else 0
             object current_step
-            bint has_apply
+            object apply_method
 
         try:
             for i in range(n):
                 current_step = steps[i]
-                has_apply = hasattr(current_step, 'apply')
-                if has_apply:
-                    value = current_step.apply(value)
+                apply_method = getattr(current_step, 'apply', None)
+                if apply_method is not None:
+                    value = apply_method(value)
                 else:
                     current_step.validate(value)
         except ValidationError:
@@ -213,17 +222,18 @@ cdef class ValidationMixin:
             return None
 
         cdef:
-            Py_ssize_t i, n
+            Py_ssize_t i, n1, n2
             BaseValidator current_validator
 
         try:
-            n = len(validators1) if validators1 else 0
-            for i in range(n):
+            n1 = len(validators1) if validators1 is not None else 0
+            n2 = len(validators2) if validators2 is not None else 0
+
+            for i in range(n1):
                 current_validator = validators1[i]
                 current_validator.validate(value)
 
-            n = len(validators2) if validators2 else 0
-            for i in range(n):
+            for i in range(n2):
                 current_validator = validators2[i]
                 current_validator.validate(value)
         except ValidationError:
@@ -267,15 +277,19 @@ cdef class ValidationMixin:
         cdef:
             dict[str, Any] validated_data = {}
             dict[str, str] errors = {}
-            Py_ssize_t i, n = len(fields) if fields else 0
-
-        cdef:
-            list field_names = list(fields.keys()) if n > 0 else []
-            list field_infos = list(fields.values()) if n > 0 else []
+            Py_ssize_t i, n = len(fields) if fields is not None else 0
+            list field_names
+            list field_infos
             str field_name
             FieldModel field_info
             object value
 
+        if n == 0:
+            return validated_data, errors
+
+        field_names = list(fields.keys())
+        field_infos = list(fields.values())
+
         for i in range(n):
             field_name = field_names[i]
             field_info = field_infos[i]

flask_inputfilter/models/base_condition/_base_condition.pyx

@@ -1,4 +1,6 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
 
 from typing import Any
 

flask_inputfilter/models/base_filter/_base_filter.pyx

@@ -1,4 +1,6 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
 
 cdef class BaseFilter:
     """

flask_inputfilter/models/base_validator/_base_validator.pyx

@@ -1,4 +1,6 @@
 # cython: language=c++
+# cython: boundscheck=False
+# cython: wraparound=False
 
 cdef class BaseValidator:
     """

flask_inputfilter/models/external_api_config/_external_api_config.pyx

@@ -1,5 +1,7 @@
 # cython: language=c++
 # cython: freelist=256
+# cython: boundscheck=False
+# cython: wraparound=False
 
 import cython
 from typing import Any

flask_inputfilter/models/field_model/_field_model.pyx

@@ -1,12 +1,15 @@
 # cython: language=c++
-# cython: freelist=256
+# cython: freelist=512
+# cython: boundscheck=False
+# cython: wraparound=False
 
 import cython
 from typing import Any
 
 from flask_inputfilter.models.cimports cimport BaseFilter, BaseValidator, ExternalApiConfig
 
 cdef list EMPTY_LIST = []
+cdef tuple EMPTY_TUPLE = ()
 
 
 @cython.final
@@ -37,8 +40,8 @@ cdef class FieldModel:
         self.required = required
         self._default = default
         self.fallback = fallback
-        self.filters = filters if filters is not None else EMPTY_LIST
-        self.validators = validators if validators is not None else EMPTY_LIST
-        self.steps = steps if steps is not None else EMPTY_LIST
+        self.filters = EMPTY_LIST if filters is None or len(filters) == 0 else filters
+        self.validators = EMPTY_LIST if validators is None or len(validators) == 0 else validators
+        self.steps = EMPTY_LIST if steps is None or len(steps) == 0 else steps
         self.external_api = external_api
         self.copy = copy