-
Notifications
You must be signed in to change notification settings - Fork 309
Adds a ser_json_datetime config option for configuring how datetimes are serialized #1465
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 9 commits
7fad743
051776e
7dcf4bb
319adfb
e9ebd11
f94636e
46ff422
f12edec
61cee77
6226f2b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -358,6 +358,7 @@ def to_json( | |||||
exclude_none: bool = False, | ||||||
round_trip: bool = False, | ||||||
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', | ||||||
datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601', | ||||||
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', | ||||||
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', | ||||||
serialize_unknown: bool = False, | ||||||
|
@@ -379,6 +380,7 @@ def to_json( | |||||
exclude_none: Whether to exclude fields that have a value of `None`. | ||||||
round_trip: Whether to enable serialization and validation round-trip support. | ||||||
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'` or `'milliseconds_float'`. | ||||||
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'` or `'milliseconds_int'`. | ||||||
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. | ||||||
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. | ||||||
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails | ||||||
|
@@ -433,6 +435,7 @@ def to_jsonable_python( | |||||
exclude_none: bool = False, | ||||||
round_trip: bool = False, | ||||||
timedelta_mode: Literal['iso8601', 'seconds_float', 'milliseconds_float'] = 'iso8601', | ||||||
datetime_mode: Literal['iso8601', 'seconds_int', 'milliseconds_int'] = 'iso8601', | ||||||
bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', | ||||||
inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', | ||||||
serialize_unknown: bool = False, | ||||||
|
@@ -454,6 +457,7 @@ def to_jsonable_python( | |||||
exclude_none: Whether to exclude fields that have a value of `None`. | ||||||
round_trip: Whether to enable serialization and validation round-trip support. | ||||||
timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_float'`, or`'milliseconds_float'`. | ||||||
datetime_mode: How to serialize `timedelta` objects, either `'iso8601'`, `'seconds_int'`, or`'milliseconds_int'`. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. | ||||||
inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. | ||||||
serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,12 +4,15 @@ use std::str::{from_utf8, FromStr, Utf8Error}; | |
use base64::Engine; | ||
use pyo3::intern; | ||
use pyo3::prelude::*; | ||
use pyo3::types::{PyDict, PyString}; | ||
use pyo3::types::{PyDateTime, PyDict, PyString}; | ||
|
||
use serde::ser::Error; | ||
|
||
use crate::build_tools::py_schema_err; | ||
use crate::input::EitherTimedelta; | ||
use crate::serializers::type_serializers::datetime_etc::{ | ||
datetime_to_milliseconds, datetime_to_seconds, datetime_to_string, | ||
}; | ||
use crate::tools::SchemaDict; | ||
|
||
use super::errors::py_err_se_err; | ||
|
@@ -18,25 +21,34 @@ use super::errors::py_err_se_err; | |
#[allow(clippy::struct_field_names)] | ||
pub(crate) struct SerializationConfig { | ||
pub timedelta_mode: TimedeltaMode, | ||
pub datetime_mode: DatetimeMode, | ||
pub bytes_mode: BytesMode, | ||
pub inf_nan_mode: InfNanMode, | ||
} | ||
|
||
impl SerializationConfig { | ||
pub fn from_config(config: Option<&Bound<'_, PyDict>>) -> PyResult<Self> { | ||
let timedelta_mode = TimedeltaMode::from_config(config)?; | ||
let datetime_mode = DatetimeMode::from_config(config)?; | ||
let bytes_mode = BytesMode::from_config(config)?; | ||
let inf_nan_mode = InfNanMode::from_config(config)?; | ||
Ok(Self { | ||
timedelta_mode, | ||
datetime_mode, | ||
bytes_mode, | ||
inf_nan_mode, | ||
}) | ||
} | ||
|
||
pub fn from_args(timedelta_mode: &str, bytes_mode: &str, inf_nan_mode: &str) -> PyResult<Self> { | ||
pub fn from_args( | ||
timedelta_mode: &str, | ||
datetime_mode: &str, | ||
bytes_mode: &str, | ||
inf_nan_mode: &str, | ||
) -> PyResult<Self> { | ||
Ok(Self { | ||
timedelta_mode: TimedeltaMode::from_str(timedelta_mode)?, | ||
datetime_mode: DatetimeMode::from_str(datetime_mode)?, | ||
bytes_mode: BytesMode::from_str(bytes_mode)?, | ||
inf_nan_mode: InfNanMode::from_str(inf_nan_mode)?, | ||
}) | ||
|
@@ -92,6 +104,14 @@ serialization_mode! { | |
MillisecondsFloat => "milliseconds_float" | ||
} | ||
|
||
serialization_mode! { | ||
DatetimeMode, | ||
"ser_json_datetime", | ||
Iso8601 => "iso8601", | ||
SecondsInt => "seconds_int", | ||
MillisecondsInt => "milliseconds_int" | ||
} | ||
|
||
serialization_mode! { | ||
BytesMode, | ||
"ser_json_bytes", | ||
|
@@ -190,6 +210,45 @@ impl BytesMode { | |
} | ||
} | ||
|
||
impl DatetimeMode { | ||
pub fn datetime_to_json(self, py: Python, datetime: &Bound<'_, PyDateTime>) -> PyResult<PyObject> { | ||
match self { | ||
Self::Iso8601 => Ok(datetime_to_string(datetime)?.into_py(py)), | ||
Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.into_py(py)), | ||
Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.into_py(py)), | ||
Comment on lines
+216
to
+218
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm, it feels a bit odd to me that we're pulling in serialization functions into our config logic, though I understand the utility here. @davidhewitt, any idea what the best approach for a refactor would be here so that we get the same behavior? I don't have any great ideas given that this generally follows the pattern that the other modes have. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is there any chance to get a SecondsStr, which would be equivalent to doing |
||
} | ||
} | ||
|
||
pub fn json_key<'py>(self, datetime: &Bound<'_, PyDateTime>) -> PyResult<Cow<'py, str>> { | ||
match self { | ||
Self::Iso8601 => Ok(datetime_to_string(datetime)?.to_string().into()), | ||
Self::SecondsInt => Ok(datetime_to_seconds(datetime)?.to_string().into()), | ||
Self::MillisecondsInt => Ok(datetime_to_milliseconds(datetime)?.to_string().into()), | ||
} | ||
} | ||
|
||
pub fn datetime_serialize<S: serde::ser::Serializer>( | ||
self, | ||
datetime: &Bound<'_, PyDateTime>, | ||
serializer: S, | ||
) -> Result<S::Ok, S::Error> { | ||
match self { | ||
Self::Iso8601 => { | ||
let s = datetime_to_string(datetime).map_err(py_err_se_err)?; | ||
serializer.serialize_str(&s) | ||
} | ||
Self::SecondsInt => { | ||
let s = datetime_to_seconds(datetime).map_err(py_err_se_err)?; | ||
serializer.serialize_i64(s) | ||
} | ||
Self::MillisecondsInt => { | ||
let s = datetime_to_milliseconds(datetime).map_err(py_err_se_err)?; | ||
serializer.serialize_i64(s) | ||
} | ||
} | ||
} | ||
} | ||
|
||
pub fn utf8_py_error(py: Python, err: Utf8Error, data: &[u8]) -> PyErr { | ||
match pyo3::exceptions::PyUnicodeDecodeError::new_utf8_bound(py, data, err) { | ||
Ok(decode_err) => PyErr::from_value_bound(decode_err.into_any()), | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -116,7 +116,7 @@ combined_serializer! { | |
Decimal: super::type_serializers::decimal::DecimalSerializer; | ||
Str: super::type_serializers::string::StrSerializer; | ||
Bytes: super::type_serializers::bytes::BytesSerializer; | ||
Datetime: super::type_serializers::datetime_etc::DatetimeSerializer; | ||
Datetime: super::type_serializers::datetime_etc::DateTimeSerializer; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's stick with |
||
TimeDelta: super::type_serializers::timedelta::TimeDeltaSerializer; | ||
Date: super::type_serializers::datetime_etc::DateSerializer; | ||
Time: super::type_serializers::datetime_etc::TimeSerializer; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,19 +3,27 @@ use std::borrow::Cow; | |
use pyo3::prelude::*; | ||
use pyo3::types::{PyDate, PyDateTime, PyDict, PyTime}; | ||
|
||
use crate::definitions::DefinitionsBuilder; | ||
use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time}; | ||
use crate::PydanticSerializationUnexpectedValue; | ||
|
||
use super::{ | ||
infer_json_key, infer_serialize, infer_to_python, py_err_se_err, BuildSerializer, CombinedSerializer, Extra, | ||
SerMode, TypeSerializer, | ||
}; | ||
use crate::definitions::DefinitionsBuilder; | ||
use crate::input::{pydate_as_date, pydatetime_as_datetime, pytime_as_time}; | ||
use crate::serializers::config::{DatetimeMode, FromConfig}; | ||
use crate::PydanticSerializationUnexpectedValue; | ||
|
||
pub(crate) fn datetime_to_string(py_dt: &Bound<'_, PyDateTime>) -> PyResult<String> { | ||
pydatetime_as_datetime(py_dt).map(|dt| dt.to_string()) | ||
} | ||
|
||
pub(crate) fn datetime_to_seconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult<i64> { | ||
pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp()) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hm, seems to be an issue here :( Looks like dt.timestamp() here doesn't take into account the number of microseconds in the datetime, so anything under a second resolution will get ignored. Looks like its a speed date thing: So i guess a few options:
🤷🏻 |
||
} | ||
|
||
pub(crate) fn datetime_to_milliseconds(py_dt: &Bound<'_, PyDateTime>) -> PyResult<i64> { | ||
pydatetime_as_datetime(py_dt).map(|dt| dt.timestamp() * 1000) | ||
} | ||
|
||
pub(crate) fn date_to_string(py_date: &Bound<'_, PyDate>) -> PyResult<String> { | ||
pydate_as_date(py_date).map(|dt| dt.to_string()) | ||
} | ||
|
@@ -118,11 +126,76 @@ macro_rules! build_serializer { | |
}; | ||
} | ||
|
||
build_serializer!( | ||
DatetimeSerializer, | ||
"datetime", | ||
PyAnyMethods::downcast::<PyDateTime>, | ||
datetime_to_string | ||
); | ||
#[derive(Debug)] | ||
pub struct DateTimeSerializer { | ||
datetime_mode: DatetimeMode, | ||
} | ||
|
||
impl BuildSerializer for DateTimeSerializer { | ||
const EXPECTED_TYPE: &'static str = "datetime"; | ||
|
||
fn build( | ||
_schema: &Bound<'_, PyDict>, | ||
config: Option<&Bound<'_, PyDict>>, | ||
_definitions: &mut DefinitionsBuilder<CombinedSerializer>, | ||
) -> PyResult<CombinedSerializer> { | ||
let datetime_mode = DatetimeMode::from_config(config)?; | ||
Ok(Self { datetime_mode }.into()) | ||
} | ||
} | ||
impl_py_gc_traverse!(DateTimeSerializer {}); | ||
|
||
impl TypeSerializer for DateTimeSerializer { | ||
fn to_python( | ||
&self, | ||
value: &Bound<'_, PyAny>, | ||
include: Option<&Bound<'_, PyAny>>, | ||
exclude: Option<&Bound<'_, PyAny>>, | ||
extra: &Extra, | ||
) -> PyResult<PyObject> { | ||
match extra.mode { | ||
SerMode::Json => match PyAnyMethods::downcast::<PyDateTime>(value) { | ||
Ok(py_value) => Ok(self.datetime_mode.datetime_to_json(value.py(), py_value)?), | ||
Err(_) => { | ||
extra.warnings.on_fallback_py(self.get_name(), value, extra)?; | ||
infer_to_python(value, include, exclude, extra) | ||
} | ||
}, | ||
_ => infer_to_python(value, include, exclude, extra), | ||
} | ||
} | ||
|
||
fn json_key<'a>(&self, key: &'a Bound<'_, PyAny>, extra: &Extra) -> PyResult<Cow<'a, str>> { | ||
match PyAnyMethods::downcast::<PyDateTime>(key) { | ||
Ok(py_value) => Ok(self.datetime_mode.json_key(py_value)?), | ||
Err(_) => { | ||
extra.warnings.on_fallback_py(self.get_name(), key, extra)?; | ||
infer_json_key(key, extra) | ||
} | ||
} | ||
} | ||
|
||
fn serde_serialize<S: serde::ser::Serializer>( | ||
&self, | ||
value: &Bound<'_, PyAny>, | ||
serializer: S, | ||
include: Option<&Bound<'_, PyAny>>, | ||
exclude: Option<&Bound<'_, PyAny>>, | ||
extra: &Extra, | ||
) -> Result<S::Ok, S::Error> { | ||
match PyAnyMethods::downcast::<PyDateTime>(value) { | ||
Ok(py_value) => self.datetime_mode.datetime_serialize(py_value, serializer), | ||
Err(_) => { | ||
extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?; | ||
infer_serialize(value, serializer, include, exclude, extra) | ||
} | ||
} | ||
} | ||
|
||
fn get_name(&self) -> &str { | ||
Self::EXPECTED_TYPE | ||
} | ||
} | ||
|
||
build_serializer!(DateSerializer, "date", downcast_date_reject_datetime, date_to_string); | ||
build_serializer!(TimeSerializer, "time", PyAnyMethods::downcast::<PyTime>, time_to_string); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.