diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index adff512c9..d58563a02 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -358,6 +358,7 @@ def to_json( exclude_none: bool = False, round_trip: bool = False, timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', + datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601', bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', serialize_unknown: bool = False, @@ -379,6 +380,7 @@ def to_json( exclude_none: Whether to exclude fields that have a value of `None`. round_trip: Whether to enable serialization and validation round-trip support. timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`. + datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'` or `'milliseconds_int'`. bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails @@ -433,6 +435,7 @@ def to_jsonable_python( exclude_none: bool = False, round_trip: bool = False, timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', + datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601', bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', serialize_unknown: bool = False, @@ -454,6 +457,7 @@ def to_jsonable_python( exclude_none: Whether to exclude fields that have a value of `None`. round_trip: Whether to enable serialization and validation round-trip support. timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`. + datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'`, or`'milliseconds_int'`. bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails diff --git a/python/pydantic_core/core_schema.py b/python/pydantic_core/core_schema.py index 922900170..8be28466d 100644 --- a/python/pydantic_core/core_schema.py +++ b/python/pydantic_core/core_schema.py @@ -67,6 +67,8 @@ class CoreConfig(TypedDict, total=False): str_to_upper: Whether to convert string fields to uppercase. allow_inf_nan: Whether to allow infinity and NaN values for float fields. Default is `True`. ser_json_timedelta: The serialization option for `timedelta` values. Default is 'iso8601'. + + ser_json_datetime: The serialization option for `datetime` values. Default is 'iso8601'. ser_json_bytes: The serialization option for `bytes` values. Default is 'utf8'. ser_json_inf_nan: The serialization option for infinity and NaN values in float fields. Default is 'null'. @@ -106,6 +108,7 @@ class CoreConfig(TypedDict, total=False): allow_inf_nan: bool # default: True # the config options are used to customise serialization to JSON ser_json_timedelta: Literal['iso8601', 'seconds_float', 'milliseconds_float'] # default: 'iso8601' + ser_json_datetime: Literal['iso8601', 'seconds_int', 'milliseconds_int'] # default: 'iso8601' ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null' val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' diff --git a/src/errors/validation_exception.rs b/src/errors/validation_exception.rs index 8d3b4eca0..c87e3dcb3 100644 --- a/src/errors/validation_exception.rs +++ b/src/errors/validation_exception.rs @@ -322,7 +322,7 @@ impl ValidationError { include_context: bool, include_input: bool, ) -> PyResult> { - let state = SerializationState::new("iso8601", "utf8", "constants")?; + let state = SerializationState::new("iso8601", "iso8601", "utf8", "constants")?; let extra = state.extra( py, &SerMode::Json, diff --git a/src/serializers/config.rs b/src/serializers/config.rs index 8b1367958..1841c4060 100644 --- a/src/serializers/config.rs +++ b/src/serializers/config.rs @@ -4,12 +4,15 @@ use std::str::{from_utf8, FromStr, Utf8Error}; use base64::Engine; use pyo3::intern; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::types::{PyDateTime, PyDict, PyString}; use serde::ser::Error; use crate::build_tools::py_schema_err; use crate::input::EitherTimedelta; +use crate::serializers::type_serializers::datetime_etc::{ + datetime_to_milliseconds, datetime_to_seconds, datetime_to_string, +}; use crate::tools::SchemaDict; use super::errors::py_err_se_err; @@ -18,6 +21,7 @@ use super::errors::py_err_se_err; #[allow(clippy::struct_field_names)] pub(crate) struct SerializationConfig { pub timedelta_mode: TimedeltaMode, + pub datetime_mode: DatetimeMode, pub bytes_mode: BytesMode, pub inf_nan_mode: InfNanMode, } @@ -25,18 +29,26 @@ pub(crate) struct SerializationConfig { impl SerializationConfig { pub fn from_config(config: Option<&Bound<'_, PyDict>>) -> PyResult { let timedelta_mode = TimedeltaMode::from_config(config)?; + let datetime_mode = DatetimeMode::from_config(config)?; let bytes_mode = BytesMode::from_config(config)?; let inf_nan_mode = InfNanMode::from_config(config)?; Ok(Self { timedelta_mode, + datetime_mode, bytes_mode, inf_nan_mode, }) } - pub fn from_args(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult { + pub fn from_args( + timedelta_mode: &str, + datetime_mode: &str, + bytes_mode: &str, + inf_nan_mode: &str, + ) -> PyResult { Ok(Self { timedelta_mode: TimedeltaMode::from_str(timedelta_mode)?, + datetime_mode: DatetimeMode::from_str(datetime_mode)?, bytes_mode: BytesMode::from_str(bytes_mode)?, inf_nan_mode: InfNanMode::from_str(inf_nan_mode)?, }) @@ -92,6 +104,14 @@ serialization_mode! { MillisecondsFloat => "milliseconds_float" } +serialization_mode! { + DatetimeMode, + "ser_json_datetime", + Iso8601 => "iso8601", + SecondsInt => "seconds_int", + MillisecondsInt => "milliseconds_int" +} + serialization_mode! { BytesMode, "ser_json_bytes", @@ -190,6 +210,45 @@ impl BytesMode { } } +impl DatetimeMode { + pub fn datetime_to_json(self, py: Python, datetime: &Bound<'_, PyDateTime>) -> PyResult { + match self { + Self::Iso8601 => Ok(datetime_to_string(datetime)?.into_py(py)), + Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.into_py(py)), + Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.into_py(py)), + } + } + + pub fn json_key<'py>(self, datetime: &Bound<'_, PyDateTime>) -> PyResult> { + match self { + Self::Iso8601 => Ok(datetime_to_string(datetime)?.to_string().into()), + Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.to_string().into()), + Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.to_string().into()), + } + } + + pub fn datetime_serialize( + self, + datetime: &Bound<'_, PyDateTime>, + serializer: S, + ) -> Result { + match self { + Self::Iso8601 => { + let s = datetime_to_string(datetime).map_err(py_err_se_err)?; + serializer.serialize_str(&s) + } + Self::SecondsInt => { + let s = datetime_to_seconds(datetime).map_err(py_err_se_err)?; + serializer.serialize_i64(s) + } + Self::MillisecondsInt => { + let s = datetime_to_milliseconds(datetime).map_err(py_err_se_err)?; + serializer.serialize_i64(s) + } + } + } +} + pub fn utf8_py_error(py: Python, err: Utf8Error, data: &[u8]) -> PyErr { match pyo3::exceptions::PyUnicodeDecodeError::new_utf8_bound(py, data, err) { Ok(decode_err) => PyErr::from_value_bound(decode_err.into_any()), diff --git a/src/serializers/extra.rs b/src/serializers/extra.rs index ba47445b5..cc38a0186 100644 --- a/src/serializers/extra.rs +++ b/src/serializers/extra.rs @@ -66,10 +66,10 @@ impl DuckTypingSerMode { } impl SerializationState { - pub fn new(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult { + pub fn new(timedelta_mode: &str, datetime_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult { let warnings = CollectWarnings::new(WarningsMode::None); let rec_guard = SerRecursionState::default(); - let config = SerializationConfig::from_args(timedelta_mode, bytes_mode, inf_nan_mode)?; + let config = SerializationConfig::from_args(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?; Ok(Self { warnings, rec_guard, diff --git a/src/serializers/infer.rs b/src/serializers/infer.rs index f03e890d5..d8dcb2399 100644 --- a/src/serializers/infer.rs +++ b/src/serializers/infer.rs @@ -171,8 +171,11 @@ pub(crate) fn infer_to_python_known( })? } ObType::Datetime => { - let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(value.downcast()?)?; - iso_dt.into_py(py) + let datetime = extra + .config + .datetime_mode + .datetime_to_json(value.py(), value.downcast()?)?; + datetime.into_py(py) } ObType::Date => { let iso_date = super::type_serializers::datetime_etc::date_to_string(value.downcast()?)?; @@ -458,9 +461,8 @@ pub(crate) fn infer_serialize_known( ObType::Set => serialize_seq!(PySet), ObType::Frozenset => serialize_seq!(PyFrozenSet), ObType::Datetime => { - let py_dt = value.downcast().map_err(py_err_se_err)?; - let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(py_dt).map_err(py_err_se_err)?; - serializer.serialize_str(&iso_dt) + let py_datetime = value.downcast().map_err(py_err_se_err)?; + extra.config.datetime_mode.datetime_serialize(py_datetime, serializer) } ObType::Date => { let py_date = value.downcast().map_err(py_err_se_err)?; @@ -637,10 +639,7 @@ pub(crate) fn infer_json_key_known<'a>( .bytes_to_string(key.py(), unsafe { py_byte_array.as_bytes() }) .map(|cow| Cow::Owned(cow.into_owned())) } - ObType::Datetime => { - let iso_dt = super::type_serializers::datetime_etc::datetime_to_string(key.downcast()?)?; - Ok(Cow::Owned(iso_dt)) - } + ObType::Datetime => extra.config.datetime_mode.json_key(key.downcast()?), ObType::Date => { let iso_date = super::type_serializers::datetime_etc::date_to_string(key.downcast()?)?; Ok(Cow::Owned(iso_date)) diff --git a/src/serializers/mod.rs b/src/serializers/mod.rs index 1a0405e2c..55fa28e29 100644 --- a/src/serializers/mod.rs +++ b/src/serializers/mod.rs @@ -242,7 +242,7 @@ impl SchemaSerializer { #[allow(clippy::too_many_arguments)] #[pyfunction] #[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = true, - exclude_none = false, round_trip = false, timedelta_mode = "iso8601", bytes_mode = "utf8", + exclude_none = false, round_trip = false, timedelta_mode = "iso8601", datetime_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None, serialize_as_any = false, context = None))] pub fn to_json( @@ -255,6 +255,7 @@ pub fn to_json( exclude_none: bool, round_trip: bool, timedelta_mode: &str, + datetime_mode: &str, bytes_mode: &str, inf_nan_mode: &str, serialize_unknown: bool, @@ -262,7 +263,7 @@ pub fn to_json( serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, ) -> PyResult { - let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?; + let state = SerializationState::new(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?; let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any); let extra = state.extra( py, @@ -285,7 +286,7 @@ pub fn to_json( #[allow(clippy::too_many_arguments)] #[pyfunction] #[pyo3(signature = (value, *, include = None, exclude = None, by_alias = true, exclude_none = false, round_trip = false, - timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None, + timedelta_mode = "iso8601", datetime_mode ="iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None, serialize_as_any = false, context = None))] pub fn to_jsonable_python( py: Python, @@ -296,6 +297,7 @@ pub fn to_jsonable_python( exclude_none: bool, round_trip: bool, timedelta_mode: &str, + datetime_mode: &str, bytes_mode: &str, inf_nan_mode: &str, serialize_unknown: bool, @@ -303,7 +305,7 @@ pub fn to_jsonable_python( serialize_as_any: bool, context: Option<&Bound<'_, PyAny>>, ) -> PyResult { - let state = SerializationState::new(timedelta_mode, bytes_mode, inf_nan_mode)?; + let state = SerializationState::new(timedelta_mode, datetime_mode, bytes_mode, inf_nan_mode)?; let duck_typing_ser_mode = DuckTypingSerMode::from_bool(serialize_as_any); let extra = state.extra( py, diff --git a/src/serializers/shared.rs b/src/serializers/shared.rs index 8eb54c837..5ecb2c745 100644 --- a/src/serializers/shared.rs +++ b/src/serializers/shared.rs @@ -116,7 +116,7 @@ combined_serializer! { Decimal: super::type_serializers::decimal::DecimalSerializer; Str: super::type_serializers::string::StrSerializer; Bytes: super::type_serializers::bytes::BytesSerializer; - Datetime: super::type_serializers::datetime_etc::DatetimeSerializer; + Datetime: super::type_serializers::datetime_etc::DateTimeSerializer; TimeDelta: super::type_serializers::timedelta::TimeDeltaSerializer; Date: super::type_serializers::datetime_etc::DateSerializer; Time: super::type_serializers::datetime_etc::TimeSerializer; diff --git a/src/serializers/type_serializers/datetime_etc.rs b/src/serializers/type_serializers/datetime_etc.rs index 957fbcf29..56170f854 100644 --- a/src/serializers/type_serializers/datetime_etc.rs +++ b/src/serializers/type_serializers/datetime_etc.rs @@ -3,19 +3,27 @@ use std::borrow::Cow; use pyo3::prelude::*; use pyo3::types::{PyDate, PyDateTime, PyDict, PyTime}; -use crate::definitions::DefinitionsBuilder; -use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time}; -use crate::PydanticSerializationUnexpectedValue; - use super::{ infer_json_key, infer_serialize, infer_to_python, py_err_se_err, BuildSerializer, CombinedSerializer, Extra, SerMode, TypeSerializer, }; +use crate::definitions::DefinitionsBuilder; +use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time}; +use crate::serializers::config::{DatetimeMode, FromConfig}; +use crate::PydanticSerializationUnexpectedValue; pub(crate) fn datetime_to_string(py_dt: &Bound<'_, PyDateTime>) -> PyResult { pydatetime_as_datetime(py_dt).map(|dt| dt.to_string()) } +pub(crate) fn datetime_to_seconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult { + pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp()) +} + +pub(crate) fn datetime_to_milliseconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult { + pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp() * 1000) +} + pub(crate) fn date_to_string(py_date: &Bound<'_, PyDate>) -> PyResult { pydate_as_date(py_date).map(|dt| dt.to_string()) } @@ -118,11 +126,76 @@ macro_rules! build_serializer { }; } -build_serializer!( - DatetimeSerializer, - "datetime", - PyAnyMethods::downcast::, - datetime_to_string -); +#[derive(Debug)] +pub struct DateTimeSerializer { + datetime_mode: DatetimeMode, +} + +impl BuildSerializer for DateTimeSerializer { + const EXPECTED_TYPE: &'static str = "datetime"; + + fn build( + _schema: &Bound<'_, PyDict>, + config: Option<&Bound<'_, PyDict>>, + _definitions: &mut DefinitionsBuilder, + ) -> PyResult { + let datetime_mode = DatetimeMode::from_config(config)?; + Ok(Self { datetime_mode }.into()) + } +} +impl_py_gc_traverse!(DateTimeSerializer {}); + +impl TypeSerializer for DateTimeSerializer { + fn to_python( + &self, + value: &Bound<'_, PyAny>, + include: Option<&Bound<'_, PyAny>>, + exclude: Option<&Bound<'_, PyAny>>, + extra: &Extra, + ) -> PyResult { + match extra.mode { + SerMode::Json => match PyAnyMethods::downcast::(value) { + Ok(py_value) => Ok(self.datetime_mode.datetime_to_json(value.py(), py_value)?), + Err(_) => { + extra.warnings.on_fallback_py(self.get_name(), value, extra)?; + infer_to_python(value, include, exclude, extra) + } + }, + _ => infer_to_python(value, include, exclude, extra), + } + } + + fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult> { + match PyAnyMethods::downcast::(key) { + Ok(py_value) => Ok(self.datetime_mode.json_key(py_value)?), + Err(_) => { + extra.warnings.on_fallback_py(self.get_name(), key, extra)?; + infer_json_key(key, extra) + } + } + } + + fn serde_serialize( + &self, + value: &Bound<'_, PyAny>, + serializer: S, + include: Option<&Bound<'_, PyAny>>, + exclude: Option<&Bound<'_, PyAny>>, + extra: &Extra, + ) -> Result { + match PyAnyMethods::downcast::(value) { + Ok(py_value) => self.datetime_mode.datetime_serialize(py_value, serializer), + Err(_) => { + extra.warnings.on_fallback_ser::(self.get_name(), value, extra)?; + infer_serialize(value, serializer, include, exclude, extra) + } + } + } + + fn get_name(&self) -> &str { + Self::EXPECTED_TYPE + } +} + build_serializer!(DateSerializer, "date", downcast_date_reject_datetime, date_to_string); build_serializer!(TimeSerializer, "time", PyAnyMethods::downcast::, time_to_string); diff --git a/tests/serializers/test_any.py b/tests/serializers/test_any.py index 9fbc8ac03..065ce0762 100644 --- a/tests/serializers/test_any.py +++ b/tests/serializers/test_any.py @@ -312,6 +312,64 @@ def test_any_config_timedelta( assert s.to_json({td: 'foo'}) == expected_to_json_dict +@pytest.mark.parametrize( + 'dt,expected_to_python,expected_to_json,expected_to_python_dict,expected_to_json_dict,mode', + [ + ( + datetime(2024, 1, 1, 0, 0, 0), + '2024-01-01T00:00:00', + b'"2024-01-01T00:00:00"', + {'2024-01-01T00:00:00': 'foo'}, + b'{"2024-01-01T00:00:00":"foo"}', + 'iso8601', + ), + ( + datetime(2024, 1, 1, 0, 0, 0), + 1704067200, + b'1704067200', + {'1704067200': 'foo'}, + b'{"1704067200":"foo"}', + 'seconds_int', + ), + ( + datetime(2024, 1, 1, 0, 0, 0), + 1704067200000, + b'1704067200000', + {'1704067200000': 'foo'}, + b'{"1704067200000":"foo"}', + 'milliseconds_int', + ), + ( + datetime( + 2024, + 1, + 1, + 0, + 0, + 0, + 1, + ), + 1704067200001, + b'1704067200001', + {'1704067200001': 'foo'}, + b'{"170406720001":"foo"}', + 'milliseconds_int', + ), + ], +) +def test_any_config_datetime( + dt: datetime, expected_to_python, expected_to_json, expected_to_python_dict, expected_to_json_dict, mode +): + s = SchemaSerializer(core_schema.any_schema(), config={'ser_json_datetime': mode}) + assert s.to_python(dt) == dt + assert s.to_python(dt, mode='json') == expected_to_python + assert s.to_json(dt) == expected_to_json + + assert s.to_python({dt: 'foo'}) == {dt: 'foo'} + assert s.to_python({dt: 'foo'}, mode='json') == expected_to_python_dict + assert s.to_json({dt: 'foo'}) == expected_to_json_dict + + def test_recursion(any_serializer): v = [1, 2] v.append(v)