[GR-60392] Update pydantic-core, make it work on windows

PullRequest: graalpython/3610

Author: msimacek

graalpython/com.oracle.graal.python.cext/CAPIFunctions.txt

@@ -356,6 +356,14 @@ PyMapping_Length;Py_ssize_t;PyObject*
 PyMapping_SetItemString;int;PyObject*|const char*|PyObject*
 PyMapping_Size;Py_ssize_t;PyObject*
 PyMapping_Values;PyObject*;PyObject*
+PyMarshal_ReadLastObjectFromFile;PyObject*;FILE*
+PyMarshal_ReadLongFromFile;long;FILE*
+PyMarshal_ReadObjectFromFile;PyObject*;FILE*
+PyMarshal_ReadObjectFromString;PyObject*;const char*|Py_ssize_t
+PyMarshal_ReadShortFromFile;int;FILE*
+PyMarshal_WriteLongToFile;void;long|FILE*|int
+PyMarshal_WriteObjectToFile;void;PyObject*|FILE*|int
+PyMarshal_WriteObjectToString;PyObject*;PyObject*|int
 PyMem_Calloc;void*;size_t|size_t
 PyMem_Free;void;void*
 PyMem_GetAllocator;void;PyMemAllocatorDomain|PyMemAllocatorEx*

graalpython/com.oracle.graal.python.cext/src/listobject.c

@@ -132,7 +132,7 @@ PyTypeObject PyList_Type = {
 };
 
 // alias for internal function, currently used in PyO3
-void _PyList_SET_ITEM(PyObject* a, Py_ssize_t b, PyObject* c) {
+PyAPI_FUNC(void) _PyList_SET_ITEM(PyObject* a, Py_ssize_t b, PyObject* c) {
     return PyList_SET_ITEM(a, b, c);
 }
 

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -2546,7 +2546,7 @@ Py_ssize_t PyTruffle_REFCNT(PyObject *obj) {
 }
 
 // alias, currently used in PyO3
-Py_ssize_t _Py_REFCNT(PyObject *obj) {
+PyAPI_FUNC(Py_ssize_t) _Py_REFCNT(PyObject *obj) {
     return PyTruffle_REFCNT(obj);
 }
 
@@ -2589,7 +2589,7 @@ PyTypeObject* PyTruffle_TYPE(PyObject *a) {
 }
 
 // alias, currently used in PyO3
-PyTypeObject* _Py_TYPE(PyObject *obj) {
+PyAPI_FUNC(PyTypeObject*) _Py_TYPE(PyObject *obj) {
     return PyTruffle_TYPE(obj);
 }
 
@@ -2625,7 +2625,7 @@ Py_ssize_t PyTruffle_SIZE(PyObject *ob) {
 }
 
 // alias, currently used in PyO3
-Py_ssize_t _Py_SIZE(PyObject *obj) {
+PyAPI_FUNC(Py_ssize_t) _Py_SIZE(PyObject *obj) {
     return PyTruffle_SIZE(obj);
 }
 

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -149,7 +149,7 @@ def from_test_case(cls, test_file: Path, test: unittest.TestCase):
                     test_id = f'<{action}>'
                 else:
                     test_id = f'{class_name}.<{action}>'
-        elif type(test).id is not unittest.TestCase.id:
+        elif type(test).id is not unittest.TestCase.id and type(test) is not unittest.FunctionTestCase:
             # Qualify doctests so that we know what they are
             test_id = f'{type(test).__qualname__}.{test_id}'
         return cls(test_file, test_id)

graalpython/com.oracle.graal.python.test/src/tests/test_repl.py

@@ -82,8 +82,8 @@ def validate_repl(stdin, python_args=(), ignore_preamble=True):
                 rlist, _, _ = select.select([pty_parent], [], [], 60)
                 assert pty_parent in rlist, f"Timed out waiting for REPL output. Output: {whole_out}{out}"
                 out += os.read(pty_parent, 1024).decode('utf-8')
-                out = out.replace('\r\n', '\n')
                 out = re.sub(r'\x1b\[(?:\?2004[hl]|\d+[A-G])', '', out)
+                out = out.replace('\r\n', '\n')
                 if out == '>>> ' or out.endswith(('\n>>> ', '\n... ')):
                     prompt = out[:3]
                     actual = out[:-5]

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java

@@ -107,6 +107,7 @@
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PY_UNICODE_PTR;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.Pointer;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PrimitiveResult32;
+import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PrimitiveResult64;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyCodeAddressRange;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyCodeObject;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyFrameObject;
@@ -799,6 +800,14 @@ public final class CApiFunction {
     @CApiBuiltin(name = "PyMapping_HasKey", ret = Int, args = {PyObject, PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "PyMapping_HasKeyString", ret = Int, args = {PyObject, ConstCharPtrAsTruffleString}, call = NotImplemented)
     @CApiBuiltin(name = "PyMapping_SetItemString", ret = Int, args = {PyObject, ConstCharPtrAsTruffleString, PyObject}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_ReadLastObjectFromFile", ret = PyObject, args = {FILE_PTR}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_ReadLongFromFile", ret = PrimitiveResult64, args = {FILE_PTR}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_ReadObjectFromFile", ret = PyObject, args = {FILE_PTR}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_ReadObjectFromString", ret = PyObject, args = {ConstCharPtr, Py_ssize_t}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_ReadShortFromFile", ret = PrimitiveResult32, args = {FILE_PTR}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_WriteLongToFile", ret = Void, args = {PrimitiveResult64, FILE_PTR, PrimitiveResult32}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_WriteObjectToFile", ret = Void, args = {PyObject, FILE_PTR, PrimitiveResult32}, call = NotImplemented)
+    @CApiBuiltin(name = "PyMarshal_WriteObjectToString", ret = PyObject, args = {PyObject, PrimitiveResult32}, call = NotImplemented)
     @CApiBuiltin(name = "PyMem_GetAllocator", ret = Void, args = {PYMEMALLOCATORDOMAIN, PYMEMALLOCATOREX_PTR}, call = NotImplemented)
     @CApiBuiltin(name = "PyMem_SetAllocator", ret = Void, args = {PYMEMALLOCATORDOMAIN, PYMEMALLOCATOREX_PTR}, call = NotImplemented)
     @CApiBuiltin(name = "PyMem_SetupDebugHooks", ret = Void, args = {}, call = NotImplemented)

graalpython/lib-graalpython/patches/metadata.toml

@@ -451,13 +451,18 @@ license = 'BSD-3-Clause'
 
 [[pydantic.rules]]
 # Pin to match our pinned pydantic-core
-version = '== 2.4.2'
+version = '== 2.10.3'
 
 [[pydantic-core.rules]]
 version = '== 2.10.1'
 patch = 'pydantic-core-2.10.1.patch'
 license = 'MIT'
 
+[[pydantic-core.rules]]
+version = '== 2.27.1'
+patch = 'pydantic-core-2.27.1.patch'
+license = 'MIT'
+
 [[pydantic-core.rules]]
 version = '== 2.26.0'
 patch = 'pydantic-core-2.26.0.patch'

graalpython/lib-graalpython/patches/pydantic-core-2.26.0.patch

@@ -37,32 +37,39 @@ index 625e474..e3ad221 100644
  dbg_macro = "warn"
 diff --git a/build_backend.py b/build_backend.py
 new file mode 100644
-index 0000000..4576d33
+index 0000000..471610f
 --- /dev/null
 +++ b/build_backend.py
-@@ -0,0 +1,23 @@
+@@ -0,0 +1,30 @@
 +import maturin
 +import os
 +import shutil
 +import subprocess
++import tarfile
++
++from pathlib import Path
++from urllib.request import urlopen
 +
 +build_sdist = maturin.build_sdist
 +
++
++def get_crate_and_patch(namespace, name, version, patched_crates):
++    with (
++        urlopen(f'https://github.com/{namespace}/{name}/archive/refs/tags/v{version}.tar.gz') as f,
++        tarfile.open(fileobj=f, mode='r:gz') as tar,
++    ):
++        tar.extractall(patched_crates, filter='data')
++    crate = patched_crates / f'{name}-{version}'
++    crate = crate.rename(patched_crates / name)
++    subprocess.check_call(['patch', '-p1', '-d', str(crate), '-i', str((Path('patches') / f'{name}.patch').absolute())])
++
++
 +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
-+    shutil.rmtree('patched_crates', ignore_errors=True)
-+    os.makedirs('patched_crates')
-+    subprocess.check_call([
-+        'git', 'clone', 'https://github.com/PyO3/pyo3.git',
-+        'patched_crates/pyo3',
-+        '--branch', 'v0.22.5',
-+    ])
-+    subprocess.check_call(['patch', '-p1', '-d', 'patched_crates/pyo3', '-i', '../../patches/pyo3.patch'])
-+    subprocess.check_call([
-+        'git', 'clone', 'https://github.com/pydantic/jiter.git',
-+        'patched_crates/jiter',
-+        '--branch', 'v0.7.0',
-+    ])
-+    subprocess.check_call(['patch', '-p1', '-d', 'patched_crates/jiter', '-i', '../../patches/jiter.patch'])
++    patched_crates = Path('patched_crates')
++    shutil.rmtree(patched_crates, ignore_errors=True)
++    patched_crates.mkdir()
++    get_crate_and_patch('PyO3', 'pyo3', '0.22.5', patched_crates)
++    get_crate_and_patch('pydantic', 'jiter', '0.7.0', patched_crates)
 +    return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
 diff --git a/patches/jiter.patch b/patches/jiter.patch
 new file mode 100644

graalpython/lib-graalpython/patches/pydantic-core-2.27.1.patch

@@ -0,0 +1,172 @@
+diff --git a/Cargo.toml b/Cargo.toml
+index 3b3b8a9..718029a 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -29,7 +29,7 @@ rust-version = "1.75"
+ [dependencies]
+ # TODO it would be very nice to remove the "py-clone" feature as it can panic,
+ # but needs a bit of work to make sure it's not used in the codebase
+-pyo3 = { version = "0.22.6", features = ["generate-import-lib", "num-bigint", "py-clone"] }
++pyo3 = { version = "0.22.6", features = ["generate-import-lib", "num-bigint", "py-clone"], path = "patched_crates/pyo3" }
+ regex = "1.11.1"
+ strum = { version = "0.26.3", features = ["derive"] }
+ strum_macros = "0.26.4"
+@@ -46,7 +46,7 @@ base64 = "0.22.1"
+ num-bigint = "0.4.6"
+ python3-dll-a = "0.2.10"
+ uuid = "1.11.0"
+-jiter = { version = "0.7.1", features = ["python"] }
++jiter = { version = "0.7.1", features = ["python"], path = "patched_crates/jiter/crates/jiter" }
+ hex = "0.4.3"
+ 
+ [lib]
+@@ -74,12 +74,12 @@ debug = true
+ strip = false
+ 
+ [dev-dependencies]
+-pyo3 = { version = "0.22.6", features = ["auto-initialize"] }
++pyo3 = { version = "0.22.6", features = ["auto-initialize"], path = "patched_crates/pyo3" }
+ 
+ [build-dependencies]
+ version_check = "0.9.5"
+ # used where logic has to be version/distribution specific, e.g. pypy
+-pyo3-build-config = { version = "0.22.6" }
++pyo3-build-config = { version = "0.22.6", path = "patched_crates/pyo3/pyo3-build-config" }
+ 
+ [lints.clippy]
+ dbg_macro = "warn"
+diff --git a/build_backend.py b/build_backend.py
+new file mode 100644
+index 0000000..4710e3f
+--- /dev/null
++++ b/build_backend.py
+@@ -0,0 +1,30 @@
++import maturin
++import os
++import shutil
++import subprocess
++import tarfile
++
++from pathlib import Path
++from urllib.request import urlopen
++
++build_sdist = maturin.build_sdist
++
++
++def get_crate_and_patch(namespace, name, version, patched_crates):
++    with (
++        urlopen(f'https://github.com/{namespace}/{name}/archive/refs/tags/v{version}.tar.gz') as f,
++        tarfile.open(fileobj=f, mode='r:gz') as tar,
++    ):
++        tar.extractall(patched_crates, filter='data')
++    crate = patched_crates / f'{name}-{version}'
++    crate = crate.rename(patched_crates / name)
++    subprocess.check_call(['patch', '-p1', '-d', str(crate), '-i', str((Path('patches') / f'{name}.patch').absolute())])
++
++
++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
++    patched_crates = Path('patched_crates')
++    shutil.rmtree(patched_crates, ignore_errors=True)
++    patched_crates.mkdir()
++    get_crate_and_patch('PyO3', 'pyo3', '0.22.6', patched_crates)
++    get_crate_and_patch('pydantic', 'jiter', '0.7.1', patched_crates)
++    return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
+diff --git a/patches/jiter.patch b/patches/jiter.patch
+new file mode 100644
+index 0000000..3fc59ee
+--- /dev/null
++++ b/patches/jiter.patch
+@@ -0,0 +1,12 @@
++diff --git a/Cargo.toml b/Cargo.toml
++index 122e111..9ec9e6d 100644
++--- a/Cargo.toml
+++++ b/Cargo.toml
++@@ -29,5 +29,5 @@ inherits = "release"
++ debug = true
++ 
++ [workspace.dependencies]
++-pyo3 = { version = "0.22.0" }
++-pyo3-build-config = { version = "0.22.0" }
+++pyo3 = { version = "0.22.6", path = "../pyo3" }
+++pyo3-build-config = { version = "0.22.6", path = "../pyo3/pyo3-build-config" }
+diff --git a/patches/pyo3.patch b/patches/pyo3.patch
+new file mode 100644
+index 0000000..e488ffb
+--- /dev/null
++++ b/patches/pyo3.patch
+@@ -0,0 +1,31 @@
++diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs
++index 6d17ebc812..1d206f9b18 100644
++--- a/pyo3-ffi/src/cpython/pyerrors.rs
+++++ b/pyo3-ffi/src/cpython/pyerrors.rs
++@@ -6,19 +6,19 @@ use crate::Py_ssize_t;
++ #[derive(Debug)]
++ pub struct PyBaseExceptionObject {
++     pub ob_base: PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub dict: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub args: *mut PyObject,
++-    #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))]
+++    #[cfg(all(Py_3_11, not(PyPy)))]
++     pub notes: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub traceback: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub context: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub cause: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub suppress_context: char,
++ }
++ 
+diff --git a/pyproject.toml b/pyproject.toml
+index de04a39..8db6756 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -3,7 +3,8 @@ requires = [
+     'maturin>=1,<2',
+     'typing-extensions >=4.6.0,!=4.7.0'
+ ]
+-build-backend = 'maturin'
++backend-path = ['.']
++build-backend = 'build_backend'
+ 
+ [project]
+ name = 'pydantic_core'
+diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs
+index c91a6e2..d0da05a 100644
+--- a/src/input/return_enums.rs
++++ b/src/input/return_enums.rs
+@@ -10,7 +10,7 @@ use pyo3::exceptions::PyTypeError;
+ use pyo3::ffi;
+ use pyo3::intern;
+ use pyo3::prelude::*;
+-#[cfg(not(PyPy))]
++#[cfg(all(not(PyPy),not(GraalPy)))]
+ use pyo3::types::PyFunction;
+ use pyo3::types::{PyBytes, PyComplex, PyFloat, PyFrozenSet, PyIterator, PyMapping, PySet, PyString};
+ 
+@@ -330,14 +330,14 @@ pub(crate) fn iterate_attributes<'a, 'py>(
+                 // the PyFunction::is_type_of(attr) catches `staticmethod`, but also any other function,
+                 // I think that's better than including static methods in the yielded attributes,
+                 // if someone really wants fields, they can use an explicit field, or a function to modify input
+-                #[cfg(not(PyPy))]
++                #[cfg(all(not(PyPy),not(GraalPy)))]
+                 if !is_bound && !attr.is_instance_of::<PyFunction>() {
+                     return Some(Ok((name, attr)));
+                 }
+                 // MASSIVE HACK! PyFunction doesn't exist for PyPy,
+                 // is_instance_of::<PyFunction> crashes with a null pointer, hence this hack, see
+                 // https://github.com/pydantic/pydantic-core/pull/161#discussion_r917257635
+-                #[cfg(PyPy)]
++                #[cfg(any(PyPy,GraalPy))]
+                 if !is_bound && attr.get_type().to_string() != "<class 'function'>" {
+                     return Some(Ok((name, attr)));
+                 }

scripts/csignature.py

@@ -63,6 +63,7 @@
 #include <frameobject.h>
 #include <datetime.h>
 #include <structmember.h>
+#include <marshal.h>
 """
 
 include_path = sysconfig.get_config_var("INCLUDEPY")