Add MISSING sentinel (#1711)

As we now require `typing-extensions>=4.14.1`, we also update pyodide to 0.27.7 to fix an issue with dependencies specification.

Author: Viicos

.github/workflows/ci.yml

@@ -329,8 +329,8 @@ jobs:
       - uses: mymindstorm/setup-emsdk@v14
         with:
           # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version
-          # in node_modules/pyodide/repodata.json, to get the version, run:
-          # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform`
+          # in node_modules/pyodide/pyodide-lock.json, to get the version, run:
+          # `cat node_modules/pyodide/pyodide-lock.json | jq .info.platform`
           version: '3.1.58'
           actions-cache-folder: emsdk-cache
 

package.json

@@ -8,7 +8,7 @@
   "main": "tests/emscripten_runner.js",
   "dependencies": {
     "prettier": "^2.7.1",
-    "pyodide": "^0.26.3"
+    "pyodide": "^0.27.7"
   },
   "scripts": {
     "test": "node tests/emscripten_runner.js",

pyproject.toml

@@ -36,7 +36,7 @@ classifiers = [
     'Typing :: Typed',
 ]
 dependencies = [
-    'typing-extensions>=4.13.0',
+    'typing-extensions>=4.14.1',
 ]
 dynamic = ['license', 'readme', 'version']
 
@@ -160,3 +160,4 @@ fix = ["create", "fix"]
 # this ensures that `uv run` doesn't actually build the package; a `make`
 # command is needed to build
 package = false
+required-version = '>=0.7.2'

python/pydantic_core/__init__.py

@@ -3,6 +3,8 @@
 import sys as _sys
 from typing import Any as _Any
 
+from typing_extensions import Sentinel
+
 from ._pydantic_core import (
     ArgsKwargs,
     MultiHostUrl,
@@ -40,6 +42,7 @@
 
 __all__ = [
     '__version__',
+    'UNSET',
     'CoreConfig',
     'CoreSchema',
     'CoreSchemaType',
@@ -140,3 +143,29 @@ class MultiHostHost(_TypedDict):
     """The host part of this host, or `None`."""
     port: int | None
     """The port part of this host, or `None`."""
+
+
+MISSING = Sentinel('MISSING')
+"""A singleton indicating a field value was not provided during validation.
+
+This singleton can be used a default value, as an alternative to `None` when it has
+an explicit meaning. During serialization, any field with `MISSING` as a value is excluded
+from the output.
+
+Example:
+    ```python
+    from pydantic import BaseModel
+
+    from pydantic_core import MISSING
+
+
+    class Configuration(BaseModel):
+        timeout: int | None | MISSING = MISSING
+
+
+    # configuration defaults, stored somewhere else:
+    defaults = {'timeout': 200}
+
+    conf = Configuration.model_validate({...})
+    timeout = conf.timeout if timeout.timeout is not MISSING else defaults['timeout']
+"""

python/pydantic_core/core_schema.py

@@ -1346,6 +1346,16 @@ class Color(Enum):
     )
 
 
+class MissingSentinelSchema(TypedDict, total=False):
+    type: Required[Literal['missing-sentinel']]
+
+
+def missing_sentinel_schema() -> MissingSentinelSchema:
+    """Returns a schema for the `MISSING` sentinel."""
+
+    return {'type': 'missing-sentinel'}
+
+
 # must match input/parse_json.rs::JsonType::try_from
 JsonType = Literal['null', 'bool', 'int', 'float', 'str', 'list', 'dict']
 
@@ -4082,6 +4092,7 @@ def definition_reference_schema(
         DatetimeSchema,
         TimedeltaSchema,
         LiteralSchema,
+        MissingSentinelSchema,
         EnumSchema,
         IsInstanceSchema,
         IsSubclassSchema,
@@ -4140,6 +4151,7 @@ def definition_reference_schema(
     'datetime',
     'timedelta',
     'literal',
+    'missing-sentinel',
     'enum',
     'is-instance',
     'is-subclass',
@@ -4239,6 +4251,7 @@ def definition_reference_schema(
     'value_error',
     'assertion_error',
     'literal_error',
+    'missing_sentinel_error',
     'date_type',
     'date_parsing',
     'date_from_datetime_parsing',

src/common/missing_sentinel.rs

@@ -0,0 +1,16 @@
+use pyo3::intern;
+use pyo3::prelude::*;
+use pyo3::sync::GILOnceCell;
+
+static MISSING_SENTINEL_OBJECT: GILOnceCell<Py<PyAny>> = GILOnceCell::new();
+
+pub fn get_missing_sentinel_object(py: Python) -> &Bound<'_, PyAny> {
+    MISSING_SENTINEL_OBJECT
+        .get_or_init(py, || {
+            py.import(intern!(py, "pydantic_core"))
+                .and_then(|core_module| core_module.getattr(intern!(py, "MISSING")))
+                .unwrap()
+                .into()
+        })
+        .bind(py)
+}

src/common/mod.rs

@@ -1,2 +1,3 @@
+pub(crate) mod missing_sentinel;
 pub(crate) mod prebuilt;
 pub(crate) mod union;

src/errors/types.rs

@@ -316,6 +316,8 @@ error_types! {
         expected: {ctx_type: String, ctx_fn: field_from_context},
     },
     // ---------------------
+    // missing sentinel
+    MissingSentinelError {},
     // date errors
     DateType {},
     DateParsing {
@@ -531,6 +533,7 @@ impl ErrorType {
             Self::AssertionError {..} => "Assertion failed, {error}",
             Self::CustomError {..} => "",  // custom errors are handled separately
             Self::LiteralError {..} => "Input should be {expected}",
+            Self::MissingSentinelError { .. } => "Input should be the 'MISSING' sentinel",
             Self::DateType {..} => "Input should be a valid date",
             Self::DateParsing {..} => "Input should be a valid date in the format YYYY-MM-DD, {error}",
             Self::DateFromDatetimeParsing {..} => "Input should be a valid date or datetime, {error}",

src/serializers/computed_fields.rs

@@ -4,6 +4,7 @@ use pyo3::{intern, PyTraverseError, PyVisit};
 use serde::ser::SerializeMap;
 
 use crate::build_tools::py_schema_error_type;
+use crate::common::missing_sentinel::get_missing_sentinel_object;
 use crate::definitions::DefinitionsBuilder;
 use crate::py_gc::PyGcTraverse;
 use crate::serializers::filter::SchemaFilter;
@@ -148,6 +149,10 @@ impl ComputedFields {
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            let missing_sentinel = get_missing_sentinel_object(model.py());
+            if value.is(missing_sentinel) {
+                continue;
+            }
 
             let field_extra = Extra {
                 field_name: Some(&computed_field.property_name),

src/serializers/fields.rs

@@ -7,6 +7,7 @@ use ahash::AHashMap;
 use serde::ser::SerializeMap;
 use smallvec::SmallVec;
 
+use crate::common::missing_sentinel::get_missing_sentinel_object;
 use crate::serializers::extra::SerCheck;
 use crate::PydanticSerializationUnexpectedValue;
 
@@ -15,8 +16,7 @@ use super::errors::py_err_se_err;
 use super::extra::Extra;
 use super::filter::SchemaFilter;
 use super::infer::{infer_json_key, infer_serialize, infer_to_python, SerializeInfer};
-use super::shared::PydanticSerializer;
-use super::shared::{CombinedSerializer, TypeSerializer};
+use super::shared::{CombinedSerializer, PydanticSerializer, TypeSerializer};
 
 /// representation of a field for serialization
 #[derive(Debug)]
@@ -169,6 +169,7 @@ impl GeneralFieldsSerializer {
     ) -> PyResult<Bound<'py, PyDict>> {
         let output_dict = PyDict::new(py);
         let mut used_req_fields: usize = 0;
+        let missing_sentinel = get_missing_sentinel_object(py);
 
         // NOTE! we maintain the order of the input dict assuming that's right
         for result in main_iter {
@@ -178,6 +179,10 @@ impl GeneralFieldsSerializer {
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            if value.is(missing_sentinel) {
+                continue;
+            }
+
             let field_extra = Extra {
                 field_name: Some(key_str),
                 ..extra
@@ -253,9 +258,13 @@ impl GeneralFieldsSerializer {
 
         for result in main_iter {
             let (key, value) = result.map_err(py_err_se_err)?;
+            let missing_sentinel = get_missing_sentinel_object(value.py());
             if extra.exclude_none && value.is_none() {
                 continue;
             }
+            if value.is(missing_sentinel) {
+                continue;
+            }
             let key_str = key_str(&key).map_err(py_err_se_err)?;
             let field_extra = Extra {
                 field_name: Some(key_str),
@@ -347,6 +356,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
         extra: &Extra,
     ) -> PyResult<PyObject> {
         let py = value.py();
+        let missing_sentinel = get_missing_sentinel_object(py);
         // If there is already a model registered (from a dataclass, BaseModel)
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
@@ -368,6 +378,9 @@ impl TypeSerializer for GeneralFieldsSerializer {
                 if extra.exclude_none && value.is_none() {
                     continue;
                 }
+                if value.is(missing_sentinel) {
+                    continue;
+                }
                 if let Some((next_include, next_exclude)) = self.filter.key_filter(&key, include, exclude)? {
                     let value = match &self.extra_serializer {
                         Some(serializer) => {
@@ -401,7 +414,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
             extra.warnings.on_fallback_ser::<S>(self.get_name(), value, extra)?;
             return infer_serialize(value, serializer, include, exclude, extra);
         };
-
+        let missing_sentinel = get_missing_sentinel_object(value.py());
         // If there is already a model registered (from a dataclass, BaseModel)
         // then do not touch it
         // If there is no model, we (a TypedDict) are the model
@@ -428,6 +441,9 @@ impl TypeSerializer for GeneralFieldsSerializer {
                 if extra.exclude_none && value.is_none() {
                     continue;
                 }
+                if value.is(missing_sentinel) {
+                    continue;
+                }
                 let filter = self.filter.key_filter(&key, include, exclude).map_err(py_err_se_err)?;
                 if let Some((next_include, next_exclude)) = filter {
                     let output_key = infer_json_key(&key, extra).map_err(py_err_se_err)?;