Improve performance by removing DjangoGetter (#28)

* Get Managers and Files working

* Reuse DjangoGetter for resolvers for now, AliasPath, formatting

* Don't attach validators for ClassVars

* Redo FileField type to use core schema

* Add resolver and alias support, add compatibility mode

* Clean up, docs

* Fix docs

* Fix 3.8 compatibility

* Add compatibility for Pydantic 2.6 and older

* Improve test coverage, bug fixes

- Made message for edge case error in FileField more useful
- Reverted wrap validator for Schema in compatibility mode to fix edge case incompatibility

* Docs, clean up

* Improve wording in docs and comments

* Revert small change to DjangoGetter

Author: pmdevita

docs/docs/differences.md

@@ -13,6 +13,34 @@ A write-up will be completed for them if Shinobi releases 1.4.0 before Ninja.
 
 ## Features
 
+### Schema Performance Improvements
+
+Shinobi significantly improves the performance of Schema, especially for handling large data payloads. 
+These improvements are not fully backwards compatible. Depending on the project, 
+they may work without any changes to your code, but you may need to make changes for 
+custom `model_validator` or `field_validator`s. There may also be issues with FileFields on Pydantic 2.6 
+and older, so upgrading is recommended.
+
+Shinobi has a compatibility mode to retain support for the old Schema behavior. This compatibility mode is 
+enabled by default in 1.4.0 to help ease the migration, and the full performance improvements are currently **opt-in**. 
+You can enable them by setting `NINJA_COMPATIBILITY` in your settings.py to False.
+
+```python
+# settings.py
+NINJA_COMPATIBILITY = False  # True by default
+```
+
+The performance improvements can also be configured per Schema by setting `_compatibility` to `True` or `False`.
+
+```python
+class MySchema(Schema):
+    _compatibility = True
+    ...
+```
+
+In 1.5.0, the default value for `NINJA_COMPATIBILITY` will be set to `True`, making the performance improvements 
+**opt-out**. The compatibility behavior will be removed in 1.6.0.
+
 ### Improved Choices Enum support
 
 [Choices and Enums](/django-shinobi/guides/response/django-pydantic/#choices-and-enums)
@@ -102,3 +130,11 @@ such as `toCamel`, Pydantic will not rewrite the alias for the foreign key field
 
 Shinobi adds a `@property` field to the Schema so that the normal name can be accessed without 
 using Pydantic's aliases, freeing it to be used for other manual or automatically generated aliases.
+
+
+### FileFields now properly validate when non-null
+
+Previously, while FileField and ImageField could show a non-null type, they would always accept 
+null. This is fixed with the schema improvements and requires Pydantic 2.7.
+Fixing this created a regression where Pydantic 2.6 and older 
+always show the field as nullable, so upgrading is recommended.

ninja/conf.py

@@ -29,6 +29,8 @@ class Settings(BaseModel):
         {"PUT", "PATCH", "DELETE"}, alias="NINJA_FIX_REQUEST_FILES_METHODS"
     )
 
+    COMPATIBILITY: bool = Field(True, alias="NINJA_COMPATIBILITY")
+
     class Config:
         from_attributes = True
 

ninja/files.py

@@ -1,10 +1,19 @@
-from typing import Any, Callable, Dict
+from typing import Callable, Dict, Union
 
 from django.core.files.uploadedfile import UploadedFile as DjangoUploadedFile
+from django.db.models.fields.files import FieldFile, FileField
+from pydantic import GetCoreSchemaHandler
 from pydantic_core import core_schema
+from typing_extensions import Annotated, Any
 
 __all__ = ["UploadedFile"]
 
+from pydantic_core.core_schema import (
+    BeforeValidatorFunctionSchema,
+    ChainSchema,
+    ValidationInfo,
+)
+
 
 class UploadedFile(DjangoUploadedFile):
     @classmethod
@@ -27,3 +36,67 @@ def __get_pydantic_core_schema__(
         cls, source: Any, handler: Callable[..., Any]
     ) -> Any:
         return core_schema.with_info_plain_validator_function(cls._validate)
+
+
+def validate_file_field(value: Any, info: ValidationInfo) -> Any:
+    if isinstance(value, FieldFile):
+        if not value:
+            return None
+        return value.url
+    return value
+
+
+class _FileFieldType:
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls,
+        _source_type: Any,
+        _handler: GetCoreSchemaHandler,
+    ) -> core_schema.CoreSchema:
+        from ninja.signature.details import is_optional
+
+        # Deprecate: Pydantic 2.6-2.7 do not support model_type_stack
+        if hasattr(_handler._generate_schema, "model_type_stack"):  # type: ignore[attr-defined]
+            # Introspect the field using this type to determine if it's supposed to be optional
+            # TODO: Make a test that creates a stack > 1
+            if (
+                len(_handler._generate_schema.model_type_stack._stack) != 1  # type: ignore[attr-defined]
+            ):
+                raise Exception(
+                    "Unexpected issue creating a schema with a FileField. Please open an issue on Django Shinobi."
+                )  # pragma: no cover
+
+            file_field_schema: Union[BeforeValidatorFunctionSchema, ChainSchema] = (
+                core_schema.chain_schema([
+                    core_schema.with_info_before_validator_function(
+                        validate_file_field, core_schema.str_schema()
+                    ),
+                    core_schema.str_schema(),
+                ])
+            )
+
+            field = _handler._generate_schema.model_type_stack._stack[  # type: ignore[attr-defined]
+                0
+            ].model_fields[_handler.field_name]
+
+            optional = is_optional(field.annotation)
+        else:
+            # Older versions of Pydantic do not return this info
+            # Set optional to True just in case (this was the old behavior anyway, we're just being more honest now)
+            optional = True
+
+        if optional:
+            field_type = core_schema.union_schema([
+                core_schema.str_schema(),
+                core_schema.none_schema(),
+            ])
+            file_field_schema = core_schema.with_info_before_validator_function(
+                validate_file_field, field_type
+            )
+
+        return core_schema.json_or_python_schema(
+            json_schema=file_field_schema, python_schema=file_field_schema
+        )
+
+
+FileFieldType = Annotated[FileField, _FileFieldType]

ninja/orm/factory.py

@@ -62,7 +62,8 @@ def create_schema(
 
         definitions = {}
         for fld in model_fields_list:
-            field_name, python_type, field_info = get_schema_field(
+            # Mypy is very confused about get_schema_field
+            field_name, python_type, field_info = get_schema_field(  # type: ignore[no-untyped-call, unused-ignore]
                 fld,
                 depth=depth,
                 optional=optional_fields and (fld.name in optional_fields),

ninja/orm/fields.py

@@ -23,6 +23,7 @@
 
 from ninja.enum import NinjaChoicesList
 from ninja.errors import ConfigError
+from ninja.files import FileFieldType
 from ninja.openapi.schema import OpenAPISchema
 from ninja.types import DictStrAny
 
@@ -67,12 +68,13 @@ def validate(cls, value: Any, _: Any) -> Any:
     "DateTimeField": datetime.datetime,
     "DecimalField": Decimal,
     "DurationField": datetime.timedelta,
-    "FileField": str,
+    "FileField": FileFieldType,
     "FilePathField": str,
     "FloatField": float,
     "GenericIPAddressField": IPvAnyAddress,
     "IPAddressField": IPvAnyAddress,
     "IntegerField": int,
+    "ImageField": FileFieldType,
     "JSONField": AnyObject,
     "NullBooleanField": bool,
     "PositiveBigIntegerField": int,