support trailing-strings with allow_partial (#1539)

Author: samuelcolvin

.mypy-stubtest-allowlist

@@ -1,7 +1,10 @@
 # TODO: don't want to expose this staticmethod, requires https://github.com/PyO3/pyo3/issues/2384
 pydantic_core._pydantic_core.PydanticUndefinedType.new
-# As per #1240, from_json has custom logic to coverage the `cache_strings` kwarg
+# See #1540 for discussion
 pydantic_core._pydantic_core.from_json
+pydantic_core._pydantic_core.SchemaValidator.validate_python
+pydantic_core._pydantic_core.SchemaValidator.validate_json
+pydantic_core._pydantic_core.SchemaValidator.validate_strings
 # the `warnings` kwarg for SchemaSerializer functions has custom logic
 pydantic_core._pydantic_core.SchemaSerializer.to_python
 pydantic_core._pydantic_core.SchemaSerializer.to_json

Cargo.lock

@@ -46,7 +46,7 @@ base64 = "0.22.1"
 num-bigint = "0.4.6"
 python3-dll-a = "0.2.10"
 uuid = "1.11.0"
-jiter = { version = "0.7", features = ["python"] }
+jiter = { version = "0.7.1", features = ["python"] }
 hex = "0.4.3"
 
 [lib]

Cargo.toml

@@ -109,3 +109,6 @@ require_change_file = false
 [tool.pyright]
 include = ['pydantic_core', 'tests/test_typing.py']
 reportUnnecessaryTypeIgnoreComment = true
+
+[tool.inline-snapshot.shortcuts]
+fix = ["create", "fix"]

benches/main.rs

@@ -96,7 +96,7 @@ class SchemaValidator:
         from_attributes: bool | None = None,
         context: Any | None = None,
         self_instance: Any | None = None,
-        allow_partial: bool = False,
+        allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False,
     ) -> Any:
         """
         Validate a Python object against the schema and return the validated object.
@@ -113,6 +113,7 @@ class SchemaValidator:
                 validation from the `__init__` method of a model.
             allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
                 and mappings are ignored.
+                `'trailing-strings'` means any final unfinished JSON string is included in the result.
 
         Raises:
             ValidationError: If validation fails.
@@ -146,7 +147,7 @@ class SchemaValidator:
         strict: bool | None = None,
         context: Any | None = None,
         self_instance: Any | None = None,
-        allow_partial: bool = False,
+        allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False,
     ) -> Any:
         """
         Validate JSON data directly against the schema and return the validated Python object.
@@ -166,6 +167,7 @@ class SchemaValidator:
             self_instance: An instance of a model set attributes on from validation.
             allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully
                 and errors in the last element of sequences and mappings are ignored.
+                `'trailing-strings'` means any final unfinished JSON string is included in the result.
 
         Raises:
             ValidationError: If validation fails or if the JSON data is invalid.
@@ -180,7 +182,7 @@ class SchemaValidator:
         *,
         strict: bool | None = None,
         context: Any | None = None,
-        allow_partial: bool = False,
+        allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False,
     ) -> Any:
         """
         Validate a string against the schema and return the validated Python object.
@@ -196,6 +198,7 @@ class SchemaValidator:
                 [`info.context`][pydantic_core.core_schema.ValidationInfo.context].
             allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences
                 and mappings are ignored.
+                `'trailing-strings'` means any final unfinished JSON string is included in the result.
 
         Raises:
             ValidationError: If validation fails or if the JSON data is invalid.
@@ -433,6 +436,7 @@ def from_json(
             `all/True` means cache all strings, `keys` means cache only dict keys, `none/False` means no caching.
         allow_partial: Whether to allow partial deserialization, if `True` JSON data is returned if the end of the
             input is reached before the full object is deserialized, e.g. `["aa", "bb", "c` would return `['aa', 'bb']`.
+            `'trailing-strings'` means any final unfinished JSON string is included in the result.
 
     Raises:
         ValueError: If deserialization fails.

pyproject.toml

@@ -3,7 +3,7 @@ use std::cmp::Ordering;
 use std::ops::Rem;
 use std::str::FromStr;
 
-use jiter::{JsonArray, JsonValue, StringCacheMode};
+use jiter::{JsonArray, JsonValue, PartialMode, StringCacheMode};
 use num_bigint::BigInt;
 
 use pyo3::exceptions::PyTypeError;
@@ -128,9 +128,13 @@ pub(crate) fn validate_iter_to_vec<'py>(
 ) -> ValResult<Vec<PyObject>> {
     let mut output: Vec<PyObject> = Vec::with_capacity(capacity);
     let mut errors: Vec<ValLineError> = Vec::new();
+    let allow_partial = state.allow_partial;
 
     for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
-        state.allow_partial = is_last_partial;
+        state.allow_partial = match is_last_partial {
+            true => allow_partial,
+            false => PartialMode::Off,
+        };
         let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?;
         match validator.validate(py, item.borrow_input(), state) {
             Ok(item) => {
@@ -202,8 +206,13 @@ pub(crate) fn validate_iter_to_set<'py>(
 ) -> ValResult<()> {
     let mut errors: Vec<ValLineError> = Vec::new();
 
+    let allow_partial = state.allow_partial;
+
     for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) {
-        state.allow_partial = is_last_partial;
+        state.allow_partial = match is_last_partial {
+            true => allow_partial,
+            false => PartialMode::Off,
+        };
         let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?;
         match validator.validate(py, item.borrow_input(), state) {
             Ok(item) => {

python/pydantic_core/_pydantic_core.pyi

@@ -45,7 +45,7 @@ impl PyUrl {
     pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
         let schema_obj = SCHEMA_DEFINITION_URL
             .get_or_init(py, || build_schema_validator(py, "url"))
-            .validate_python(py, url, None, None, None, None, false)?;
+            .validate_python(py, url, None, None, None, None, false.into())?;
         schema_obj.extract(py)
     }
 
@@ -225,7 +225,7 @@ impl PyMultiHostUrl {
     pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult<Self> {
         let schema_obj = SCHEMA_DEFINITION_MULTI_HOST_URL
             .get_or_init(py, || build_schema_validator(py, "multi-host-url"))
-            .validate_python(py, url, None, None, None, None, false)?;
+            .validate_python(py, url, None, None, None, None, false.into())?;
         schema_obj.extract(py)
     }
 

src/input/return_enums.rs

@@ -11,7 +11,6 @@ use crate::build_tools::{schema_or_config_same, ExtraBehavior};
 use crate::errors::{ErrorTypeDefaults, ValError, ValLineError, ValResult};
 use crate::input::{Arguments, BorrowInput, Input, KeywordArgs, PositionalArgs, ValidationMatch};
 use crate::lookup_key::LookupKey;
-
 use crate::tools::SchemaDict;
 
 use super::validation_state::ValidationState;
@@ -189,7 +188,7 @@ impl Validator for ArgumentsValidator {
         state: &mut ValidationState<'_, 'py>,
     ) -> ValResult<PyObject> {
         // this validator does not yet support partial validation, disable it to avoid incorrect results
-        state.allow_partial = false;
+        state.allow_partial = false.into();
 
         let args = input.validate_args()?;
 

src/url.rs

@@ -146,7 +146,7 @@ impl Validator for DataclassArgsValidator {
         state: &mut ValidationState<'_, 'py>,
     ) -> ValResult<PyObject> {
         // this validator does not yet support partial validation, disable it to avoid incorrect results
-        state.allow_partial = false;
+        state.allow_partial = false.into();
 
         let args = input.validate_dataclass_args(&self.dataclass_name)?;