Add serialize_as_any runtime flag support (#1194)

Co-authored-by: David Hewitt <[email protected]>

Author: sydney-runkle

python/pydantic_core/_pydantic_core.pyi

@@ -266,6 +266,7 @@ class SchemaSerializer:
         round_trip: bool = False,
         warnings: bool = True,
         fallback: Callable[[Any], Any] | None = None,
+        serialize_as_any: bool = False,
         context: dict[str, Any] | None = None,
     ) -> Any:
         """
@@ -286,6 +287,7 @@ class SchemaSerializer:
             warnings: Whether to log warnings when invalid fields are encountered.
             fallback: A function to call when an unknown value is encountered,
                 if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised.
+            serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
             context: The context to use for serialization, this is passed to functional serializers as
                 [`info.context`][pydantic_core.core_schema.SerializationInfo.context].
 
@@ -309,6 +311,7 @@ class SchemaSerializer:
         round_trip: bool = False,
         warnings: bool = True,
         fallback: Callable[[Any], Any] | None = None,
+        serialize_as_any: bool = False,
         context: dict[str, Any] | None = None,
     ) -> bytes:
         """
@@ -328,6 +331,7 @@ class SchemaSerializer:
             warnings: Whether to log warnings when invalid fields are encountered.
             fallback: A function to call when an unknown value is encountered,
                 if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised.
+            serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
             context: The context to use for serialization, this is passed to functional serializers as
                 [`info.context`][pydantic_core.core_schema.SerializationInfo.context].
 
@@ -352,6 +356,7 @@ def to_json(
     inf_nan_mode: Literal['null', 'constants'] = 'constants',
     serialize_unknown: bool = False,
     fallback: Callable[[Any], Any] | None = None,
+    serialize_as_any: bool = False,
     context: dict[str, Any] | None = None,
 ) -> bytes:
     """
@@ -374,6 +379,7 @@ def to_json(
             `"<Unserializable {value_type} object>"` will be used.
         fallback: A function to call when an unknown value is encountered,
             if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised.
+        serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
         context: The context to use for serialization, this is passed to functional serializers as
             [`info.context`][pydantic_core.core_schema.SerializationInfo.context].
 
@@ -416,6 +422,7 @@ def to_jsonable_python(
     inf_nan_mode: Literal['null', 'constants'] = 'constants',
     serialize_unknown: bool = False,
     fallback: Callable[[Any], Any] | None = None,
+    serialize_as_any: bool = False,
     context: dict[str, Any] | None = None,
 ) -> Any:
     """
@@ -438,6 +445,7 @@ def to_jsonable_python(
             `"<Unserializable {value_type} object>"` will be used.
         fallback: A function to call when an unknown value is encountered,
             if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised.
+        serialize_as_any: Whether to serialize fields with duck-typing serialization behavior.
         context: The context to use for serialization, this is passed to functional serializers as
             [`info.context`][pydantic_core.core_schema.SerializationInfo.context].
 

python/pydantic_core/core_schema.py

@@ -142,6 +142,8 @@ def exclude_defaults(self) -> bool: ...
     def exclude_none(self) -> bool: ...
 
     @property
+    def serialize_as_any(self) -> bool: ...
+
     def round_trip(self) -> bool: ...
 
     def mode_is_json(self) -> bool: ...

src/errors/validation_exception.rs

@@ -17,7 +17,7 @@ use crate::build_tools::py_schema_error_type;
 use crate::errors::LocItem;
 use crate::get_pydantic_version;
 use crate::input::InputType;
-use crate::serializers::{SerMode, SerializationState};
+use crate::serializers::{DuckTypingSerMode, Extra, SerMode, SerializationState};
 use crate::tools::{safe_repr, SchemaDict};
 
 use super::line_error::ValLineError;
@@ -323,7 +323,17 @@ impl ValidationError {
         include_input: bool,
     ) -> PyResult<Bound<'py, PyString>> {
         let state = SerializationState::new("iso8601", "utf8", "constants")?;
-        let extra = state.extra(py, &SerMode::Json, true, false, false, true, None, None);
+        let extra = state.extra(
+            py,
+            &SerMode::Json,
+            true,
+            false,
+            false,
+            true,
+            None,
+            DuckTypingSerMode::SchemaBased,
+            None,
+        );
         let serializer = ValidationErrorSerializer {
             py,
             line_errors: &self.line_errors,
@@ -604,7 +614,7 @@ struct ValidationErrorSerializer<'py> {
     url_prefix: Option<&'py str>,
     include_context: bool,
     include_input: bool,
-    extra: &'py crate::serializers::Extra<'py>,
+    extra: &'py Extra<'py>,
     input_type: &'py InputType,
 }
 
@@ -636,7 +646,7 @@ struct PyLineErrorSerializer<'py> {
     url_prefix: Option<&'py str>,
     include_context: bool,
     include_input: bool,
-    extra: &'py crate::serializers::Extra<'py>,
+    extra: &'py Extra<'py>,
     input_type: &'py InputType,
 }
 

src/serializers/extra.rs

@@ -23,6 +23,45 @@ pub(crate) struct SerializationState {
     config: SerializationConfig,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DuckTypingSerMode {
+    // Don't check the type of the value, use the type of the schema
+    SchemaBased,
+    // Check the type of the value, use the type of the value
+    NeedsInference,
+    // We already checked the type of the value
+    // we don't want to infer again, but if we recurse down
+    // we do want to flip this back to NeedsInference for the
+    // fields / keys / items of any inner serializers
+    Inferred,
+}
+
+impl DuckTypingSerMode {
+    pub fn from_bool(serialize_as_any: bool) -> Self {
+        if serialize_as_any {
+            DuckTypingSerMode::NeedsInference
+        } else {
+            DuckTypingSerMode::SchemaBased
+        }
+    }
+
+    pub fn to_bool(self) -> bool {
+        match self {
+            DuckTypingSerMode::SchemaBased => false,
+            DuckTypingSerMode::NeedsInference => true,
+            DuckTypingSerMode::Inferred => true,
+        }
+    }
+
+    pub fn next_mode(self) -> Self {
+        match self {
+            DuckTypingSerMode::SchemaBased => DuckTypingSerMode::SchemaBased,
+            DuckTypingSerMode::NeedsInference => DuckTypingSerMode::Inferred,
+            DuckTypingSerMode::Inferred => DuckTypingSerMode::NeedsInference,
+        }
+    }
+}
+
 impl SerializationState {
     pub fn new(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult<Self> {
         let warnings = CollectWarnings::new(false);
@@ -44,8 +83,9 @@ impl SerializationState {
         exclude_none: bool,
         round_trip: bool,
         serialize_unknown: bool,
-        fallback: Option<&'py Bound<'py, PyAny>>,
-        context: Option<&'py Bound<'py, PyAny>>,
+        fallback: Option<&'py Bound<'_, PyAny>>,
+        duck_typing_ser_mode: DuckTypingSerMode,
+        context: Option<&'py Bound<'_, PyAny>>,
     ) -> Extra<'py> {
         Extra::new(
             py,
@@ -60,6 +100,7 @@ impl SerializationState {
             &self.rec_guard,
             serialize_unknown,
             fallback,
+            duck_typing_ser_mode,
             context,
         )
     }
@@ -92,6 +133,7 @@ pub(crate) struct Extra<'a> {
     pub field_name: Option<&'a str>,
     pub serialize_unknown: bool,
     pub fallback: Option<&'a Bound<'a, PyAny>>,
+    pub duck_typing_ser_mode: DuckTypingSerMode,
     pub context: Option<&'a Bound<'a, PyAny>>,
 }
 
@@ -110,6 +152,7 @@ impl<'a> Extra<'a> {
         rec_guard: &'a SerRecursionState,
         serialize_unknown: bool,
         fallback: Option<&'a Bound<'a, PyAny>>,
+        duck_typing_ser_mode: DuckTypingSerMode,
         context: Option<&'a Bound<'a, PyAny>>,
     ) -> Self {
         Self {
@@ -128,6 +171,7 @@ impl<'a> Extra<'a> {
             field_name: None,
             serialize_unknown,
             fallback,
+            duck_typing_ser_mode,
             context,
         }
     }
@@ -187,6 +231,7 @@ pub(crate) struct ExtraOwned {
     field_name: Option<String>,
     serialize_unknown: bool,
     pub fallback: Option<PyObject>,
+    duck_typing_ser_mode: DuckTypingSerMode,
     pub context: Option<PyObject>,
 }
 
@@ -206,8 +251,9 @@ impl ExtraOwned {
             model: extra.model.map(|model| model.clone().into()),
             field_name: extra.field_name.map(ToString::to_string),
             serialize_unknown: extra.serialize_unknown,
-            fallback: extra.fallback.map(|fallback| fallback.clone().into()),
-            context: extra.context.map(|context| context.clone().into()),
+            fallback: extra.fallback.map(|model| model.clone().into()),
+            duck_typing_ser_mode: extra.duck_typing_ser_mode,
+            context: extra.context.map(|model| model.clone().into()),
         }
     }
 
@@ -228,6 +274,7 @@ impl ExtraOwned {
             field_name: self.field_name.as_deref(),
             serialize_unknown: self.serialize_unknown,
             fallback: self.fallback.as_ref().map(|m| m.bind(py)),
+            duck_typing_ser_mode: self.duck_typing_ser_mode,
             context: self.context.as_ref().map(|m| m.bind(py)),
         }
     }

src/serializers/fields.rs

@@ -8,6 +8,7 @@ use serde::ser::SerializeMap;
 use smallvec::SmallVec;
 
 use crate::serializers::extra::SerCheck;
+use crate::serializers::DuckTypingSerMode;
 use crate::PydanticSerializationUnexpectedValue;
 
 use super::computed_fields::ComputedFields;
@@ -322,15 +323,30 @@ impl TypeSerializer for GeneralFieldsSerializer {
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
         let model = extra.model.map_or_else(|| Some(value), Some);
-        let td_extra = Extra { model, ..*extra };
+
+        // If there is no model, use duck typing ser logic for TypedDict
+        // If there is a model, skip this step, as BaseModel and dataclass duck typing
+        // is handled in their respective serializers
+        if extra.model.is_none() {
+            let duck_typing_ser_mode = extra.duck_typing_ser_mode.next_mode();
+            let td_extra = Extra {
+                model,
+                duck_typing_ser_mode,
+                ..*extra
+            };
+            if td_extra.duck_typing_ser_mode == DuckTypingSerMode::Inferred {
+                return infer_to_python(value, include, exclude, &td_extra);
+            }
+        }
         let (main_dict, extra_dict) = if let Some(main_extra_dict) = self.extract_dicts(value) {
             main_extra_dict
         } else {
-            td_extra.warnings.on_fallback_py(self.get_name(), value, &td_extra)?;
-            return infer_to_python(value, include, exclude, &td_extra);
+            extra.warnings.on_fallback_py(self.get_name(), value, extra)?;
+            return infer_to_python(value, include, exclude, extra);
         };
 
-        let output_dict = self.main_to_python(py, dict_items(&main_dict), include, exclude, td_extra)?;
+        let output_dict =
+            self.main_to_python(py, dict_items(&main_dict), include, exclude, Extra { model, ..*extra })?;
 
         // this is used to include `__pydantic_extra__` in serialization on models
         if let Some(extra_dict) = extra_dict {
@@ -376,7 +392,21 @@ impl TypeSerializer for GeneralFieldsSerializer {
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
         let model = extra.model.map_or_else(|| Some(value), Some);
-        let td_extra = Extra { model, ..*extra };
+
+        // If there is no model, use duck typing ser logic for TypedDict
+        // If there is a model, skip this step, as BaseModel and dataclass duck typing
+        // is handled in their respective serializers
+        if extra.model.is_none() {
+            let duck_typing_ser_mode = extra.duck_typing_ser_mode.next_mode();
+            let td_extra = Extra {
+                model,
+                duck_typing_ser_mode,
+                ..*extra
+            };
+            if td_extra.duck_typing_ser_mode == DuckTypingSerMode::Inferred {
+                return infer_serialize(value, serializer, include, exclude, &td_extra);
+            }
+        }
         let expected_len = match self.mode {
             FieldsMode::TypedDictAllow => main_dict.len() + self.computed_field_count(),
             _ => self.fields.len() + option_length!(extra_dict) + self.computed_field_count(),
@@ -389,7 +419,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
             serializer,
             include,
             exclude,
-            td_extra,
+            Extra { model, ..*extra },
         )?;
 
         // this is used to include `__pydantic_extra__` in serialization on models

src/serializers/infer.rs

@@ -103,6 +103,7 @@ pub(crate) fn infer_to_python_known(
             extra.rec_guard,
             extra.serialize_unknown,
             extra.fallback,
+            extra.duck_typing_ser_mode,
             extra.context,
         );
         serializer.serializer.to_python(value, include, exclude, &extra)
@@ -475,6 +476,7 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
                 extra.rec_guard,
                 extra.serialize_unknown,
                 extra.fallback,
+                extra.duck_typing_ser_mode,
                 extra.context,
             );
             let pydantic_serializer =