From ac47d5f84f151dfe269f4d8a5af1fbdb8abe6e3e Mon Sep 17 00:00:00 2001 From: anoa's Codex Agent Date: Wed, 24 Sep 2025 19:08:09 +0100 Subject: [PATCH] Drop pydantic v1 compatibility --- .github/workflows/tests.yml | 22 - docs/changelogs/CHANGES-2024.md | 6 +- docs/upgrade.md | 8 + poetry.lock | 48 +-- pyproject.toml | 7 +- scripts-dev/check_pydantic_models.py | 478 --------------------- scripts-dev/lint.sh | 3 - synapse/_pydantic_compat.py | 104 ----- synapse/api/auth/mas.py | 15 +- synapse/config/_util.py | 5 +- synapse/config/mas.py | 24 +- synapse/config/workers.py | 8 +- synapse/events/validator.py | 2 +- synapse/http/servlet.py | 21 +- synapse/rest/admin/users.py | 2 +- synapse/rest/client/account.py | 2 +- synapse/rest/client/devices.py | 10 +- synapse/rest/client/directory.py | 2 +- synapse/rest/client/reporting.py | 2 +- synapse/rest/key/v2/remote_key_resource.py | 5 +- synapse/rest/synapse/mas/devices.py | 2 +- synapse/rest/synapse/mas/users.py | 8 +- synapse/storage/background_updates.py | 2 +- synapse/types/handlers/sliding_sync.py | 16 +- synapse/types/rest/__init__.py | 4 +- synapse/types/rest/client/__init__.py | 35 +- synapse/util/events.py | 8 +- synapse/util/pydantic_models.py | 26 +- tests/rest/client/test_models.py | 2 +- 29 files changed, 133 insertions(+), 744 deletions(-) delete mode 100755 scripts-dev/check_pydantic_models.py delete mode 100644 synapse/_pydantic_compat.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ad171e240fe..3a06c5dddfb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -207,26 +207,6 @@ jobs: env: PULL_REQUEST_NUMBER: ${{ github.event.number }} - lint-pydantic: - runs-on: ubuntu-latest - needs: changes - if: ${{ needs.changes.outputs.linting == 'true' }} - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Install Rust - uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # master - with: - toolchain: ${{ env.RUST_VERSION }} - - uses: Swatinem/rust-cache@f13886b937689c021905a6b90929199931d60db1 # v2.8.1 - - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 - with: - poetry-version: "2.1.1" - extras: "all" - - run: poetry run scripts-dev/check_pydantic_models.py - lint-clippy: runs-on: ubuntu-latest needs: changes @@ -341,7 +321,6 @@ jobs: - lint-mypy - lint-crlf - lint-newsfile - - lint-pydantic - check-sampleconfig - check-schema-delta - check-lockfile @@ -363,7 +342,6 @@ jobs: lint lint-mypy lint-newsfile - lint-pydantic lint-clippy lint-clippy-nightly lint-rust diff --git a/docs/changelogs/CHANGES-2024.md b/docs/changelogs/CHANGES-2024.md index ee354f15733..41eb7c7b47a 100644 --- a/docs/changelogs/CHANGES-2024.md +++ b/docs/changelogs/CHANGES-2024.md @@ -375,10 +375,8 @@ No significant changes since 1.116.0rc2. ### Internal Changes -- Import pydantic objects from the `_pydantic_compat` module. - This allows `check_pydantic_models.py` to mock those pydantic objects - only in the synapse module, and not interfere with pydantic objects in - external dependencies. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) +- Drop the `_pydantic_compat` shim in favour of importing directly from pydantic v2 and + remove the `check_pydantic_models.py` helper script. ([\#17667](https://github.com/element-hq/synapse/issues/17667)) - Use [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync tables as a bulk shortcut for getting the max `event_stream_ordering` of rooms. ([\#17693](https://github.com/element-hq/synapse/issues/17693)) - Speed up [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync requests a bit where there are many room changes. ([\#17696](https://github.com/element-hq/synapse/issues/17696)) - Refactor [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) sliding sync filter unit tests so the sliding sync API has better test coverage. ([\#17703](https://github.com/element-hq/synapse/issues/17703)) diff --git a/docs/upgrade.md b/docs/upgrade.md index 5f998e9708d..152938a3c78 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -117,6 +117,14 @@ each upgrade are complete before moving on to the next upgrade, to avoid stacking them up. You can monitor the currently running background updates with [the Admin API](usage/administration/admin_api/background_updates.html#status). +# Upgrading to v1.140.0 + +## Pydantic v2 is now required + +Synapse no longer supports pydantic v1. Ensure that the Python environment used to run +Synapse installs pydantic version 2.0.0 or later (but still below 3.0.0) before +upgrading. Environments that pin dependencies should be updated accordingly. + # Upgrading to v1.139.0 ## `/register` requests from old application service implementations may break when using MAS diff --git a/poetry.lock b/poetry.lock index 8759b43dbe5..4e0e836f88e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.4-py2.py3-none-any.whl", hash = "sha256:39313d2a2caac3ecf6d8f95fbebdfd30ae6ea6ae6a6db794d976405fdd9aa796"}, {file = "authlib-1.6.4.tar.gz", hash = "sha256:104b0442a43061dc8bc23b133d1d06a2b0a9c2e3e33f34c4338929e816287649"}, @@ -435,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -460,7 +460,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -511,7 +511,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -848,7 +848,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -986,7 +986,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1002,7 +1002,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, @@ -1243,7 +1243,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1482,7 +1482,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1688,7 +1688,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1709,7 +1709,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1725,7 +1725,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1984,7 +1984,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2044,7 +2044,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2069,7 +2069,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2097,7 +2097,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2463,7 +2463,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.34.1-py2.py3-none-any.whl", hash = "sha256:b7a072e1cdc5abc48101d5146e1ae680fa81fe886d8d95aaa25a0b450c818d32"}, {file = "sentry_sdk-2.34.1.tar.gz", hash = "sha256:69274eb8c5c38562a544c3e9f68b5be0a43be4b697f5fd385bf98e4fbe672687"}, @@ -2651,7 +2651,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2667,7 +2667,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2729,7 +2729,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2866,7 +2866,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3112,7 +3112,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3256,4 +3256,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "2e8ea085e1a0c6f0ac051d4bc457a96827d01f621b1827086de01a5ffa98cf79" +content-hash = "8783bfa1c998c4cf854e173b3f6745b0e21e655e0c24a8f9cda4be5d7375dc19" diff --git a/pyproject.toml b/pyproject.toml index c548a652e90..590bf1a1fba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -222,9 +222,8 @@ ijson = ">=3.1.4" matrix-common = "^1.3.0" # We need packaging.verison.Version(...).major added in 20.0. packaging = ">=20.0" -# We support pydantic v1 and pydantic v2 via the pydantic.v1 compat module. -# See https://github.com/matrix-org/synapse/issues/15858 -pydantic = ">=1.7.4, <3" +# Synapse now requires pydantic v2. +pydantic = ">=2.0.0, <3" # This is for building the rust components during "poetry install", which # currently ignores the `build-system.requires` directive (c.f. @@ -325,7 +324,7 @@ all = [ # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. ruff = "0.12.10" -# Type checking only works with the pydantic.v1 compat module from pydantic v2 +# Type checking uses pydantic v2 APIs. pydantic = "^2" # Typechecking diff --git a/scripts-dev/check_pydantic_models.py b/scripts-dev/check_pydantic_models.py deleted file mode 100755 index 26a473a61b6..00000000000 --- a/scripts-dev/check_pydantic_models.py +++ /dev/null @@ -1,478 +0,0 @@ -#! /usr/bin/env python -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2022 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# -""" -A script which enforces that Synapse always uses strict types when defining a Pydantic -model. - -Pydantic does not yet offer a strict mode, but it is planned for pydantic v2. See - - https://github.com/pydantic/pydantic/issues/1098 - https://pydantic-docs.helpmanual.io/blog/pydantic-v2/#strict-mode - -until then, this script is a best effort to stop us from introducing type coersion bugs -(like the infamous stringy power levels fixed in room version 10). -""" - -import argparse -import contextlib -import functools -import importlib -import logging -import os -import pkgutil -import sys -import textwrap -import traceback -import unittest.mock -from contextlib import contextmanager -from typing import ( - Any, - Callable, - Dict, - Generator, - List, - Set, - Type, - TypeVar, -) - -from parameterized import parameterized -from typing_extensions import ParamSpec - -from synapse._pydantic_compat import ( - BaseModel as PydanticBaseModel, - conbytes, - confloat, - conint, - constr, - get_args, -) - -logger = logging.getLogger(__name__) - -CONSTRAINED_TYPE_FACTORIES_WITH_STRICT_FLAG: List[Callable] = [ - constr, - conbytes, - conint, - confloat, -] - -TYPES_THAT_PYDANTIC_WILL_COERCE_TO = [ - str, - bytes, - int, - float, - bool, -] - - -P = ParamSpec("P") -R = TypeVar("R") - - -class ModelCheckerException(Exception): - """Dummy exception. Allows us to detect unwanted types during a module import.""" - - -class MissingStrictInConstrainedTypeException(ModelCheckerException): - factory_name: str - - def __init__(self, factory_name: str): - self.factory_name = factory_name - - -class FieldHasUnwantedTypeException(ModelCheckerException): - message: str - - def __init__(self, message: str): - self.message = message - - -def make_wrapper(factory: Callable[P, R]) -> Callable[P, R]: - """We patch `constr` and friends with wrappers that enforce strict=True.""" - - @functools.wraps(factory) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - if "strict" not in kwargs: - raise MissingStrictInConstrainedTypeException(factory.__name__) - if not kwargs["strict"]: - raise MissingStrictInConstrainedTypeException(factory.__name__) - return factory(*args, **kwargs) - - return wrapper - - -def field_type_unwanted(type_: Any) -> bool: - """Very rough attempt to detect if a type is unwanted as a Pydantic annotation. - - At present, we exclude types which will coerce, or any generic type involving types - which will coerce.""" - logger.debug("Is %s unwanted?") - if type_ in TYPES_THAT_PYDANTIC_WILL_COERCE_TO: - logger.debug("yes") - return True - logger.debug("Maybe. Subargs are %s", get_args(type_)) - rv = any(field_type_unwanted(t) for t in get_args(type_)) - logger.debug("Conclusion: %s %s unwanted", type_, "is" if rv else "is not") - return rv - - -class PatchedBaseModel(PydanticBaseModel): - """A patched version of BaseModel that inspects fields after models are defined. - - We complain loudly if we see an unwanted type. - - Beware: ModelField.type_ is presumably private; this is likely to be very brittle. - """ - - @classmethod - def __init_subclass__(cls: Type[PydanticBaseModel], **kwargs: object): - for field in cls.__fields__.values(): - # Note that field.type_ and field.outer_type are computed based on the - # annotation type, see pydantic.fields.ModelField._type_analysis - if field_type_unwanted(field.outer_type_): - # TODO: this only reports the first bad field. Can we find all bad ones - # and report them all? - raise FieldHasUnwantedTypeException( - f"{cls.__module__}.{cls.__qualname__} has field '{field.name}' " - f"with unwanted type `{field.outer_type_}`" - ) - - -@contextmanager -def monkeypatch_pydantic() -> Generator[None, None, None]: - """Patch pydantic with our snooping versions of BaseModel and the con* functions. - - If the snooping functions see something they don't like, they'll raise a - ModelCheckingException instance. - """ - with contextlib.ExitStack() as patches: - # Most Synapse code ought to import the patched objects directly from - # `pydantic`. But we also patch their containing modules `pydantic.main` and - # `pydantic.types` for completeness. - patch_basemodel = unittest.mock.patch( - "synapse._pydantic_compat.BaseModel", new=PatchedBaseModel - ) - patches.enter_context(patch_basemodel) - for factory in CONSTRAINED_TYPE_FACTORIES_WITH_STRICT_FLAG: - wrapper: Callable = make_wrapper(factory) - patch = unittest.mock.patch( - f"synapse._pydantic_compat.{factory.__name__}", new=wrapper - ) - patches.enter_context(patch) - yield - - -def format_model_checker_exception(e: ModelCheckerException) -> str: - """Work out which line of code caused e. Format the line in a human-friendly way.""" - # TODO. FieldHasUnwantedTypeException gives better error messages. Can we ditch the - # patches of constr() etc, and instead inspect fields to look for ConstrainedStr - # with strict=False? There is some difficulty with the inheritance hierarchy - # because StrictStr < ConstrainedStr < str. - if isinstance(e, FieldHasUnwantedTypeException): - return e.message - elif isinstance(e, MissingStrictInConstrainedTypeException): - frame_summary = traceback.extract_tb(e.__traceback__)[-2] - return ( - f"Missing `strict=True` from {e.factory_name}() call \n" - + traceback.format_list([frame_summary])[0].lstrip() - ) - else: - raise ValueError(f"Unknown exception {e}") from e - - -def lint() -> int: - """Try to import all of Synapse and see if we spot any Pydantic type coercions. - - Print any problems, then return a status code suitable for sys.exit.""" - failures = do_lint() - if failures: - print(f"Found {len(failures)} problem(s)") - for failure in sorted(failures): - print(failure) - return os.EX_DATAERR if failures else os.EX_OK - - -def do_lint() -> Set[str]: - """Try to import all of Synapse and see if we spot any Pydantic type coercions.""" - failures = set() - - with monkeypatch_pydantic(): - logger.debug("Importing synapse") - try: - # TODO: make "synapse" an argument so we can target this script at - # a subpackage - module = importlib.import_module("synapse") - except ModelCheckerException as e: - logger.warning("Bad annotation found when importing synapse") - failures.add(format_model_checker_exception(e)) - return failures - - try: - logger.debug("Fetching subpackages") - module_infos = list( - pkgutil.walk_packages(module.__path__, f"{module.__name__}.") - ) - except ModelCheckerException as e: - logger.warning("Bad annotation found when looking for modules to import") - failures.add(format_model_checker_exception(e)) - return failures - - for module_info in module_infos: - logger.debug("Importing %s", module_info.name) - try: - importlib.import_module(module_info.name) - except ModelCheckerException as e: - logger.warning( - "Bad annotation found when importing %s", module_info.name - ) - failures.add(format_model_checker_exception(e)) - - return failures - - -def run_test_snippet(source: str) -> None: - """Exec a snippet of source code in an isolated environment.""" - # To emulate `source` being called at the top level of the module, - # the globals and locals we provide apparently have to be the same mapping. - # - # > Remember that at the module level, globals and locals are the same dictionary. - # > If exec gets two separate objects as globals and locals, the code will be - # > executed as if it were embedded in a class definition. - globals_: Dict[str, object] - locals_: Dict[str, object] - globals_ = locals_ = {} - exec(textwrap.dedent(source), globals_, locals_) - - -class TestConstrainedTypesPatch(unittest.TestCase): - def test_expression_without_strict_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import constr - except ImportError: - from pydantic import constr - constr() - """ - ) - - def test_called_as_module_attribute_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - import pydantic - pydantic.constr() - """ - ) - - def test_wildcard_import_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import * - except ImportError: - from pydantic import * - constr() - """ - ) - - def test_alternative_import_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1.types import constr - except ImportError: - from pydantic.types import constr - constr() - """ - ) - - def test_alternative_import_attribute_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import types as pydantic_types - except ImportError: - from pydantic import types as pydantic_types - pydantic_types.constr() - """ - ) - - def test_kwarg_but_no_strict_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import constr - except ImportError: - from pydantic import constr - constr(min_length=10) - """ - ) - - def test_kwarg_strict_False_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import constr - except ImportError: - from pydantic import constr - constr(strict=False) - """ - ) - - def test_kwarg_strict_True_doesnt_raise(self) -> None: - with monkeypatch_pydantic(): - run_test_snippet( - """ - try: - from pydantic.v1 import constr - except ImportError: - from pydantic import constr - constr(strict=True) - """ - ) - - def test_annotation_without_strict_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import constr - except ImportError: - from pydantic import constr - x: constr() - """ - ) - - def test_field_annotation_without_strict_raises(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1 import BaseModel, conint - except ImportError: - from pydantic import BaseModel, conint - class C: - x: conint() - """ - ) - - -class TestFieldTypeInspection(unittest.TestCase): - @parameterized.expand( - [ - ("str",), - ("bytes"), - ("int",), - ("float",), - ("bool"), - ("Optional[str]",), - ("Union[None, str]",), - ("List[str]",), - ("List[List[str]]",), - ("Dict[StrictStr, str]",), - ("Dict[str, StrictStr]",), - ("TypedDict('D', x=int)",), - ] - ) - def test_field_holding_unwanted_type_raises(self, annotation: str) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - f""" - from typing import * - try: - from pydantic.v1 import * - except ImportError: - from pydantic import * - class C(BaseModel): - f: {annotation} - """ - ) - - @parameterized.expand( - [ - ("StrictStr",), - ("StrictBytes"), - ("StrictInt",), - ("StrictFloat",), - ("StrictBool"), - ("constr(strict=True, min_length=10)",), - ("Optional[StrictStr]",), - ("Union[None, StrictStr]",), - ("List[StrictStr]",), - ("List[List[StrictStr]]",), - ("Dict[StrictStr, StrictStr]",), - ("TypedDict('D', x=StrictInt)",), - ] - ) - def test_field_holding_accepted_type_doesnt_raise(self, annotation: str) -> None: - with monkeypatch_pydantic(): - run_test_snippet( - f""" - from typing import * - try: - from pydantic.v1 import * - except ImportError: - from pydantic import * - class C(BaseModel): - f: {annotation} - """ - ) - - def test_field_holding_str_raises_with_alternative_import(self) -> None: - with monkeypatch_pydantic(), self.assertRaises(ModelCheckerException): - run_test_snippet( - """ - try: - from pydantic.v1.main import BaseModel - except ImportError: - from pydantic.main import BaseModel - class C(BaseModel): - f: str - """ - ) - - -parser = argparse.ArgumentParser() -parser.add_argument("mode", choices=["lint", "test"], default="lint", nargs="?") -parser.add_argument("-v", "--verbose", action="store_true") - - -if __name__ == "__main__": - args = parser.parse_args(sys.argv[1:]) - logging.basicConfig( - format="%(asctime)s %(name)s:%(lineno)d %(levelname)s %(message)s", - level=logging.DEBUG if args.verbose else logging.INFO, - ) - # suppress logs we don't care about - logging.getLogger("xmlschema").setLevel(logging.WARNING) - if args.mode == "lint": - sys.exit(lint()) - elif args.mode == "test": - unittest.main(argv=sys.argv[:1]) diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 7096100a3ef..d5e10d42926 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -134,9 +134,6 @@ fi # Ensure the formatting of Rust code. cargo-fmt -# Ensure all Pydantic models use strict types. -./scripts-dev/check_pydantic_models.py lint - # Ensure type hints are correct. mypy diff --git a/synapse/_pydantic_compat.py b/synapse/_pydantic_compat.py deleted file mode 100644 index a520c0e8971..00000000000 --- a/synapse/_pydantic_compat.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2023 Maxwell G -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from typing import TYPE_CHECKING - -from packaging.version import Version - -try: - from pydantic import __version__ as pydantic_version -except ImportError: - import importlib.metadata - - pydantic_version = importlib.metadata.version("pydantic") - -HAS_PYDANTIC_V2: bool = Version(pydantic_version).major == 2 - -if TYPE_CHECKING or HAS_PYDANTIC_V2: - from pydantic.v1 import ( - AnyHttpUrl, - BaseModel, - Extra, - Field, - FilePath, - MissingError, - PydanticValueError, - StrictBool, - StrictInt, - StrictStr, - ValidationError, - conbytes, - confloat, - conint, - constr, - parse_obj_as, - root_validator, - validator, - ) - from pydantic.v1.error_wrappers import ErrorWrapper - from pydantic.v1.typing import get_args -else: - from pydantic import ( - AnyHttpUrl, - BaseModel, - Extra, - Field, - FilePath, - MissingError, - PydanticValueError, - StrictBool, - StrictInt, - StrictStr, - ValidationError, - conbytes, - confloat, - conint, - constr, - parse_obj_as, - root_validator, - validator, - ) - from pydantic.error_wrappers import ErrorWrapper - from pydantic.typing import get_args - -__all__ = ( - "HAS_PYDANTIC_V2", - "AnyHttpUrl", - "BaseModel", - "constr", - "conbytes", - "conint", - "confloat", - "ErrorWrapper", - "Extra", - "Field", - "FilePath", - "get_args", - "MissingError", - "parse_obj_as", - "PydanticValueError", - "StrictBool", - "StrictInt", - "StrictStr", - "ValidationError", - "validator", - "root_validator", -) diff --git a/synapse/api/auth/mas.py b/synapse/api/auth/mas.py index ef82ea9cc77..3a6117e920f 100644 --- a/synapse/api/auth/mas.py +++ b/synapse/api/auth/mas.py @@ -16,14 +16,7 @@ from typing import TYPE_CHECKING, Optional, Set from urllib.parse import urlencode -from synapse._pydantic_compat import ( - BaseModel, - Extra, - StrictBool, - StrictInt, - StrictStr, - ValidationError, -) +from pydantic import BaseModel, ConfigDict, StrictBool, StrictInt, StrictStr, ValidationError from synapse.api.auth.base import BaseAuth from synapse.api.errors import ( AuthError, @@ -64,8 +57,7 @@ class ServerMetadata(BaseModel): - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", strict=True) issuer: StrictStr account_management_uri: StrictStr @@ -80,8 +72,7 @@ class IntrospectionResponse(BaseModel): device_id: Optional[StrictStr] expires_in: Optional[StrictInt] - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", strict=True) def get_scope_set(self) -> set[str]: if not self.scope: diff --git a/synapse/config/_util.py b/synapse/config/_util.py index 731b60a8407..f465053b3c9 100644 --- a/synapse/config/_util.py +++ b/synapse/config/_util.py @@ -22,7 +22,7 @@ import jsonschema -from synapse._pydantic_compat import BaseModel, ValidationError, parse_obj_as +from pydantic import BaseModel, TypeAdapter, ValidationError from synapse.config._base import ConfigError from synapse.types import JsonDict, StrSequence @@ -93,7 +93,8 @@ def parse_and_validate_mapping( try: # type-ignore: mypy doesn't like constructing `Dict[str, model_type]` because # `model_type` is a runtime variable. Pydantic is fine with this. - instances = parse_obj_as(Dict[str, model_type], config) # type: ignore[valid-type] + adapter = TypeAdapter(Dict[str, model_type]) # type: ignore[valid-type] + instances = adapter.validate_python(config) except ValidationError as e: raise ConfigError(str(e)) from e return instances diff --git a/synapse/config/mas.py b/synapse/config/mas.py index fe0d326f7af..ed5bfc0db87 100644 --- a/synapse/config/mas.py +++ b/synapse/config/mas.py @@ -15,14 +15,14 @@ from typing import Any, Optional -from synapse._pydantic_compat import ( +from pydantic import ( AnyHttpUrl, Field, FilePath, StrictBool, StrictStr, ValidationError, - validator, + model_validator, ) from synapse.config.experimental import read_secret_from_file_once from synapse.types import JsonDict @@ -37,23 +37,17 @@ class MasConfigModel(ParseModel): secret: Optional[StrictStr] = Field(default=None) secret_path: Optional[FilePath] = Field(default=None) - @validator("secret") - def validate_secret_is_set_if_enabled(cls, v: Any, values: dict) -> Any: - if values.get("enabled", False) and not values.get("secret_path") and not v: - raise ValueError( - "You must set a `secret` or `secret_path` when enabling Matrix Authentication Service integration." - ) + @model_validator(mode="after") + def validate_secret_configuration(cls, values: "MasConfigModel") -> "MasConfigModel": + if values.secret and values.secret_path: + raise ValueError("`secret` and `secret_path` cannot be set at the same time.") - return v - - @validator("secret_path") - def validate_secret_path_is_set_if_enabled(cls, v: Any, values: dict) -> Any: - if values.get("secret"): + if values.enabled and not values.secret and not values.secret_path: raise ValueError( - "`secret` and `secret_path` cannot be set at the same time." + "You must set a `secret` or `secret_path` when enabling Matrix Authentication Service integration." ) - return v + return values class MasConfig(Config): diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 825ba784820..3af9084594a 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -26,11 +26,7 @@ import attr -from synapse._pydantic_compat import ( - StrictBool, - StrictInt, - StrictStr, -) +from pydantic import StrictBool, StrictInt, StrictStr from synapse.config._base import ( Config, ConfigError, @@ -49,7 +45,7 @@ _DEPRECATED_WORKER_DUTY_OPTION_USED = """ The '%s' configuration option is deprecated and will be removed in a future -Synapse version. Please use ``%s: name_of_worker`` instead. +Synapse version. Please use `%s: name_of_worker` instead. """ _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA = """ diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 4d9ba15829e..746c402a0be 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -23,7 +23,7 @@ import jsonschema -from synapse._pydantic_compat import Field, StrictBool, StrictStr +from pydantic import Field, StrictBool, StrictStr from synapse.api.constants import ( MAX_ALIAS_LENGTH, EventContentFields, diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 71e809b3f1c..65b150b2cb5 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -40,13 +40,7 @@ from twisted.web.server import Request -from synapse._pydantic_compat import ( - BaseModel, - ErrorWrapper, - MissingError, - PydanticValueError, - ValidationError, -) +from pydantic import BaseModel, ValidationError from synapse.api.errors import Codes, SynapseError from synapse.http import redact_uri from synapse.http.server import HttpServer @@ -908,12 +902,15 @@ def validate_json_object(content: JsonDict, model_type: Type[Model]) -> Model: # clear-cut: BAD_JSON arguably overlaps with MISSING_PARAM and INVALID_PARAM. errcode = Codes.BAD_JSON - raw_errors = e.raw_errors - if len(raw_errors) == 1 and isinstance(raw_errors[0], ErrorWrapper): - raw_error = raw_errors[0].exc - if isinstance(raw_error, MissingError): + errors = e.errors() + if len(errors) == 1: + error_type = errors[0].get("type") + if error_type == "missing": errcode = Codes.MISSING_PARAM - elif isinstance(raw_error, PydanticValueError): + elif error_type and ( + error_type.startswith("value_error") + or error_type.startswith("type_error") + ): errcode = Codes.INVALID_PARAM raise SynapseError(HTTPStatus.BAD_REQUEST, str(e), errcode=errcode) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 25a38dc4acb..6cbc6f771a9 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -27,7 +27,7 @@ import attr -from synapse._pydantic_compat import StrictBool, StrictInt, StrictStr +from pydantic import StrictBool, StrictInt, StrictStr from synapse.api.constants import Direction from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index d9f0c169e80..b01d5f1bbb5 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -28,7 +28,7 @@ from twisted.web.server import Request -from synapse._pydantic_compat import StrictBool, StrictStr, constr +from pydantic import StrictBool, StrictStr, constr from synapse.api.constants import LoginType from synapse.api.errors import ( Codes, diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 0777abde7f6..be79e4ec2d4 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -24,7 +24,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple -from synapse._pydantic_compat import Extra, StrictStr +from pydantic import ConfigDict, StrictStr from synapse.api import errors from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError from synapse.http.server import HttpServer @@ -244,11 +244,10 @@ async def on_PUT( class DehydratedDeviceDataModel(RequestBodyModel): """JSON blob describing a dehydrated device to be stored. - Expects other freeform fields. Use .dict() to access them. + Expects other freeform fields. Use `.dict()` to access them. """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", frozen=True, strict=True) algorithm: StrictStr @@ -540,8 +539,7 @@ class PutBody(RequestBodyModel): device_id: StrictStr initial_device_display_name: Optional[StrictStr] - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", frozen=True, strict=True) async def on_PUT(self, request: SynapseRequest) -> Tuple[int, JsonDict]: submission = parse_and_validate_json_object_from_request(request, self.PutBody) diff --git a/synapse/rest/client/directory.py b/synapse/rest/client/directory.py index 479f489623b..f352dcb93ee 100644 --- a/synapse/rest/client/directory.py +++ b/synapse/rest/client/directory.py @@ -24,7 +24,7 @@ from twisted.web.server import Request -from synapse._pydantic_compat import StrictStr +from pydantic import StrictStr from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import ( diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 81faf38a7f8..13cacb336a6 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -23,7 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Tuple -from synapse._pydantic_compat import StrictStr +from pydantic import StrictStr from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import ( diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 94c679b9e75..410ba7a7160 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -27,7 +27,7 @@ from twisted.web.server import Request -from synapse._pydantic_compat import Extra, StrictInt, StrictStr +from pydantic import ConfigDict, StrictInt, StrictStr from synapse.crypto.keyring import ServerKeyFetcher from synapse.http.server import HttpServer from synapse.http.servlet import ( @@ -48,8 +48,7 @@ class _KeyQueryCriteriaDataModel(RequestBodyModel): - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", frozen=True, strict=True) minimum_valid_until_ts: Optional[StrictInt] diff --git a/synapse/rest/synapse/mas/devices.py b/synapse/rest/synapse/mas/devices.py index 6cc11535906..583a5a7ee21 100644 --- a/synapse/rest/synapse/mas/devices.py +++ b/synapse/rest/synapse/mas/devices.py @@ -17,7 +17,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Optional, Tuple -from synapse._pydantic_compat import StrictStr +from pydantic import StrictStr from synapse.api.errors import NotFoundError from synapse.http.servlet import parse_and_validate_json_object_from_request from synapse.types import JsonDict, UserID diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py index 09aa13bebbc..229db42a169 100644 --- a/synapse/rest/synapse/mas/users.py +++ b/synapse/rest/synapse/mas/users.py @@ -17,7 +17,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Any, Optional, Tuple, TypedDict -from synapse._pydantic_compat import StrictBool, StrictStr, root_validator +from pydantic import StrictBool, StrictStr, model_validator from synapse.api.errors import NotFoundError, SynapseError from synapse.http.servlet import ( parse_and_validate_json_object_from_request, @@ -111,8 +111,12 @@ class PostBody(RequestBodyModel): unset_emails: StrictBool = False set_emails: Optional[list[StrictStr]] = None - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def validate_exclusive(cls, values: Any) -> Any: + if not isinstance(values, dict): + return values + if "unset_displayname" in values and "set_displayname" in values: raise ValueError( "Cannot specify both unset_displayname and set_displayname" diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 9aa9e51aeb6..96bbdddd1f2 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -40,7 +40,7 @@ import attr -from synapse._pydantic_compat import BaseModel +from pydantic import BaseModel from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine from synapse.storage.types import Connection, Cursor diff --git a/synapse/types/handlers/sliding_sync.py b/synapse/types/handlers/sliding_sync.py index b7bc565464f..f53bf627c84 100644 --- a/synapse/types/handlers/sliding_sync.py +++ b/synapse/types/handlers/sliding_sync.py @@ -37,7 +37,7 @@ import attr -from synapse._pydantic_compat import Extra +from pydantic import ConfigDict from synapse.api.constants import EventTypes from synapse.events import EventBase from synapse.types import ( @@ -70,14 +70,12 @@ class SlidingSyncConfig(SlidingSyncBody): user: UserID requester: Requester - # Pydantic config - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False - # Allow custom types like `UserID` to be used in the model - arbitrary_types_allowed = True + model_config = ConfigDict( + extra="ignore", + frozen=True, + strict=True, + arbitrary_types_allowed=True, + ) class OperationType(Enum): diff --git a/synapse/types/rest/__init__.py b/synapse/types/rest/__init__.py index a02836deee1..8be0f217faa 100644 --- a/synapse/types/rest/__init__.py +++ b/synapse/types/rest/__init__.py @@ -18,8 +18,10 @@ # [This file includes modifications made by New Vector Limited] # # +from pydantic import ConfigDict + from synapse.util.pydantic_models import ParseModel class RequestBodyModel(ParseModel): - pass + model_config = ConfigDict(extra="ignore", frozen=True, strict=True) diff --git a/synapse/types/rest/client/__init__.py b/synapse/types/rest/client/__init__.py index 11d7e59b43a..b7744c39958 100644 --- a/synapse/types/rest/client/__init__.py +++ b/synapse/types/rest/client/__init__.py @@ -20,15 +20,16 @@ # from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union -from synapse._pydantic_compat import ( - Extra, +from pydantic import ( + ConfigDict, Field, StrictBool, StrictInt, StrictStr, conint, constr, - validator, + field_validator, + model_validator, ) from synapse.types.rest import RequestBodyModel from synapse.util.threepids import validate_email @@ -44,8 +45,7 @@ class AuthenticationData(RequestBodyModel): `.dict(exclude_unset=True)` to access them. """ - class Config: - extra = Extra.allow + model_config = ConfigDict(extra="allow", frozen=True, strict=True) session: Optional[StrictStr] = None type: Optional[StrictStr] = None @@ -56,7 +56,7 @@ class Config: else: # See also assert_valid_client_secret() ClientSecretStr = constr( - regex="[0-9a-zA-Z.=_-]", # noqa: F722 + pattern="[0-9a-zA-Z.=_-]", # noqa: F722 min_length=1, max_length=255, strict=True, @@ -70,13 +70,13 @@ class ThreepidRequestTokenBody(RequestBodyModel): next_link: Optional[StrictStr] send_attempt: StrictInt - @validator("id_access_token", always=True) + @model_validator(mode="after") def token_required_for_identity_server( - cls, token: Optional[str], values: Dict[str, object] - ) -> Optional[str]: - if values.get("id_server") is not None and token is None: + cls, body: "ThreepidRequestTokenBody" + ) -> "ThreepidRequestTokenBody": + if body.id_server is not None and body.id_access_token is None: raise ValueError("id_access_token is required if an id_server is supplied.") - return token + return body class EmailRequestTokenBody(ThreepidRequestTokenBody): @@ -87,14 +87,17 @@ class EmailRequestTokenBody(ThreepidRequestTokenBody): # know the exact spelling (eg. upper and lower case) of address in the database. # Without this, an email stored in the database as "foo@bar.com" would cause # user requests for "FOO@bar.com" to raise a Not Found error. - _email_validator = validator("email", allow_reuse=True)(validate_email) + @field_validator("email") + @classmethod + def _email_validator(cls, value: str) -> str: + return validate_email(value) if TYPE_CHECKING: ISO3116_1_Alpha_2 = StrictStr else: # Per spec: two-letter uppercase ISO-3166-1-alpha-2 - ISO3116_1_Alpha_2 = constr(regex="[A-Z]{2}", strict=True) + ISO3116_1_Alpha_2 = constr(pattern="[A-Z]{2}", strict=True) class MsisdnRequestTokenBody(ThreepidRequestTokenBody): @@ -286,7 +289,8 @@ class ToDeviceExtension(RequestBodyModel): limit: StrictInt = 100 since: Optional[StrictStr] = None - @validator("since") + @field_validator("since") + @classmethod def since_token_check( cls, value: Optional[StrictStr] ) -> Optional[StrictStr]: @@ -397,7 +401,8 @@ class ThreadSubscriptionsExtension(RequestBodyModel): room_subscriptions: Optional[Dict[StrictStr, RoomSubscription]] = None extensions: Optional[Extensions] = None - @validator("lists") + @field_validator("lists") + @classmethod def lists_length_check( cls, value: Optional[Dict[str, SlidingSyncList]] ) -> Optional[Dict[str, SlidingSyncList]]: diff --git a/synapse/util/events.py b/synapse/util/events.py index 4808268702a..102d76ebb9d 100644 --- a/synapse/util/events.py +++ b/synapse/util/events.py @@ -15,7 +15,7 @@ from typing import Any, List, Optional -from synapse._pydantic_compat import Field, StrictStr, ValidationError, validator +from pydantic import Field, StrictStr, ValidationError, field_validator from synapse.types import JsonDict from synapse.util.pydantic_models import ParseModel from synapse.util.stringutils import random_string @@ -60,7 +60,8 @@ class MTopic(ParseModel): # Because "Receivers SHOULD use the first representation in the array that they # understand.", we ignore invalid representations in the `m.text` field and use # what we can. - @validator("m_text", pre=True) + @field_validator("m_text", mode="before") + @classmethod def ignore_invalid_representations( cls, m_text: Any ) -> Optional[List[MTextRepresentation]]: @@ -92,7 +93,8 @@ class TopicContent(ParseModel): # We ignore invalid `m.topic` fields as we can always fall back to the plain-text # `topic` field. - @validator("m_topic", pre=True) + @field_validator("m_topic", mode="before") + @classmethod def ignore_invalid_m_topic(cls, m_topic: Any) -> Optional[MTopic]: try: return MTopic.parse_obj(m_topic) diff --git a/synapse/util/pydantic_models.py b/synapse/util/pydantic_models.py index 4880709501b..a7488c1aaba 100644 --- a/synapse/util/pydantic_models.py +++ b/synapse/util/pydantic_models.py @@ -14,9 +14,10 @@ # import re -from typing import Any, Callable, Generator +from typing import Any -from synapse._pydantic_compat import BaseModel, Extra, StrictStr +from pydantic import BaseModel, ConfigDict, GetCoreSchemaHandler +from pydantic_core import core_schema from synapse.types import EventID @@ -36,14 +37,10 @@ class ParseModel(BaseModel): https://pydantic-docs.helpmanual.io/usage/model_config/#change-behaviour-globally """ - class Config: - # By default, ignore fields that we don't recognise. - extra = Extra.ignore - # By default, don't allow fields to be reassigned after parsing. - allow_mutation = False + model_config = ConfigDict(extra="ignore", frozen=True) -class AnyEventId(StrictStr): +class AnyEventId(str): """ A validator for strings that need to be an Event ID. @@ -55,9 +52,16 @@ class AnyEventId(StrictStr): ) @classmethod - def __get_validators__(cls) -> Generator[Callable[..., Any], Any, Any]: - yield from super().__get_validators__() # type: ignore - yield cls.validate_event_id + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + str_schema = core_schema.str_schema(strict=True) + return core_schema.chain_schema( + [ + str_schema, + core_schema.no_info_plain_validator_function(cls.validate_event_id), + ] + ) @classmethod def validate_event_id(cls, value: str) -> str: diff --git a/tests/rest/client/test_models.py b/tests/rest/client/test_models.py index 75479e6235c..b68fb491e6d 100644 --- a/tests/rest/client/test_models.py +++ b/tests/rest/client/test_models.py @@ -21,7 +21,7 @@ import unittest as stdlib_unittest from typing import Literal -from synapse._pydantic_compat import BaseModel, ValidationError +from pydantic import BaseModel, ValidationError from synapse.types.rest.client import EmailRequestTokenBody