[GR-59936] Support pydantic-core 2.26

msimacek · msimacek · commit 9a245a1f0e7d · 2024-11-25T16:12:23.000Z
PullRequest: graalpython/3585
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_long.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_long.py
@@ -471,3 +471,16 @@ class TestPyLong(CPyExtTestCase):
         arguments=["const char* bytes", "Py_ssize_t size", "int little_endian", "int is_signed"],
         cmpfunc=unhandled_error_compare,
     )
+
+    test__PyLong_NumBits = CPyExtFunction(
+        lambda args: args[0].bit_length(),
+        lambda: (
+            (1,),
+            (1230948701328090743,),
+            (-1230948701328090743,),
+        ),
+        resultspec="n",
+        argspec="O",
+        arguments=["PyObject* obj"],
+        cmpfunc=unhandled_error_compare,
+    )
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_unicode.py b/graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_unicode.py
@@ -1132,6 +1132,16 @@ class TestPyUnicode(CPyExtTestCase):
         cmpfunc=unhandled_error_compare,
     )
 
+    test_PyUnicodeDecodeError_Create = CPyExtFunction(
+        lambda args: UnicodeDecodeError(*args),
+        lambda: (
+            ("utf-8", b"asdf", 1, 2, "some reason"),
+        ),
+        resultspec="O",
+        argspec="sy#nns",
+        arguments=["const char* encoding", "const char* object", "Py_ssize_t length", "Py_ssize_t start", "Py_ssize_t end", "const char* reason"]
+    )
+
 
 class TestUnicodeObject(unittest.TestCase):
     def test_intern(self):
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextLongBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextLongBuiltins.java
@@ -483,4 +483,13 @@ static Object convert(Object charPtr, long size, int littleEndian, int signed,
             return fromByteArray.execute(inliningTarget, bytes, littleEndian != 0, signed != 0);
         }
     }
+
+    @CApiBuiltin(ret = SIZE_T, args = {PyObject}, call = Direct)
+    abstract static class _PyLong_NumBits extends CApiUnaryBuiltinNode {
+        @Specialization
+        static long numBits(Object obj,
+                        @Cached IntBuiltins.BitLengthNode bitLengthNode) {
+            return bitLengthNode.execute(obj);
+        }
+    }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextUnicodeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextUnicodeBuiltins.java
@@ -1177,7 +1177,7 @@ static Object doNull(@SuppressWarnings("unused") PNone noValue) {
     @CApiBuiltin(ret = PyObjectTransfer, args = {ConstCharPtrAsTruffleString, ConstCharPtr, Py_ssize_t, Py_ssize_t, Py_ssize_t, ConstCharPtrAsTruffleString}, call = Direct)
     abstract static class PyUnicodeDecodeError_Create extends CApi6BuiltinNode {
         @Specialization
-        static Object doit(Object encoding, Object object, int length, int start, int end, Object reason,
+        static Object doit(Object encoding, Object object, long length, long start, long end, Object reason,
                         @Bind("this") Node inliningTarget,
                         @Cached GetByteArrayNode getByteArrayNode,
                         @Cached CallNode callNode,
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java
@@ -1076,7 +1076,6 @@ public final class CApiFunction {
     @CApiBuiltin(name = "_PyLong_GCD", ret = PyObject, args = {PyObject, PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "_PyLong_Lshift", ret = PyObject, args = {PyObject, SIZE_T}, call = NotImplemented)
     @CApiBuiltin(name = "_PyLong_New", ret = PyLongObject, args = {Py_ssize_t}, call = NotImplemented)
-    @CApiBuiltin(name = "_PyLong_NumBits", ret = SIZE_T, args = {PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "_PyLong_Rshift", ret = PyObject, args = {PyObject, SIZE_T}, call = NotImplemented)
     @CApiBuiltin(name = "_PyLong_Size_t_Converter", ret = Int, args = {PyObject, Pointer}, call = NotImplemented)
     @CApiBuiltin(name = "_PyLong_UnsignedInt_Converter", ret = Int, args = {PyObject, Pointer}, call = NotImplemented)
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/ExternalFunctionNodes.java
@@ -1379,13 +1379,18 @@ protected Object[] prepareCArguments(VirtualFrame frame) {
             Object[] args = readVarargsNode.executeObjectArray(frame);
             PKeyword[] kwargs = readKwargsNode.executePKeyword(frame);
             Object[] fastcallArgs = new Object[args.length + kwargs.length];
-            Object[] fastcallKwnames = new Object[kwargs.length];
+            Object kwnamesTuple = PNone.NO_VALUE;
             PythonUtils.arraycopy(args, 0, fastcallArgs, 0, args.length);
-            for (int i = 0; i < kwargs.length; i++) {
-                fastcallKwnames[i] = kwargs[i].getName();
-                fastcallArgs[args.length + i] = kwargs[i].getValue();
+            // Note: PyO3 doesn't like it when we put an empty tuple there if there are no args
+            if (kwargs.length > 0) {
+                Object[] fastcallKwnames = new Object[kwargs.length];
+                for (int i = 0; i < kwargs.length; i++) {
+                    fastcallKwnames[i] = kwargs[i].getName();
+                    fastcallArgs[args.length + i] = kwargs[i].getValue();
+                }
+                kwnamesTuple = factory.createTuple(fastcallKwnames);
             }
-            return new Object[]{self, new CPyObjectArrayWrapper(fastcallArgs), args.length, factory.createTuple(fastcallKwnames)};
+            return new Object[]{self, new CPyObjectArrayWrapper(fastcallArgs), args.length, kwnamesTuple};
         }
 
         @Override
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/CoroutineBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/CoroutineBuiltins.java
@@ -41,6 +41,7 @@
 package com.oracle.graal.python.builtins.objects.generator;
 
 import static com.oracle.graal.python.nodes.SpecialMethodNames.J___AWAIT__;
+import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REPR__;
 
 import java.util.List;
 
@@ -49,6 +50,8 @@
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
 import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.PythonAbstractObject;
+import com.oracle.graal.python.builtins.objects.str.StringUtils.SimpleTruffleStringFormatNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
@@ -60,6 +63,7 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.profiles.InlinedConditionProfile;
+import com.oracle.truffle.api.strings.TruffleString;
 
 @CoreFunctions(extendClasses = PythonBuiltinClassType.PCoroutine)
 public final class CoroutineBuiltins extends PythonBuiltins {
@@ -127,4 +131,14 @@ static Object await(PGenerator self,
             return factory.createCoroutineWrapper(self);
         }
     }
+
+    @Builtin(name = J___REPR__, minNumOfPositionalArgs = 1)
+    @GenerateNodeFactory
+    abstract static class ReprNode extends PythonUnaryBuiltinNode {
+        @Specialization
+        static TruffleString repr(PGenerator self,
+                        @Cached SimpleTruffleStringFormatNode simpleTruffleStringFormatNode) {
+            return simpleTruffleStringFormatNode.format("<coroutine object %s at 0x%d>", self.getQualname(), PythonAbstractObject.objectHashCode(self));
+        }
+    }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java
@@ -224,7 +224,7 @@ abstract static class ReprNode extends PythonUnaryBuiltinNode {
         @Specialization
         static TruffleString repr(PGenerator self,
                         @Cached SimpleTruffleStringFormatNode simpleTruffleStringFormatNode) {
-            return simpleTruffleStringFormatNode.format("<generator object %s at %d>", self.getName(), PythonAbstractObject.objectHashCode(self));
+            return simpleTruffleStringFormatNode.format("<generator object %s at 0x%d>", self.getQualname(), PythonAbstractObject.objectHashCode(self));
         }
     }
 
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java
@@ -3141,7 +3141,9 @@ static int bitCount(PInt i) {
     @Builtin(name = "bit_length", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     @TypeSystemReference(PythonArithmeticTypes.class)
-    abstract static class BitLengthNode extends PythonBuiltinNode {
+    public abstract static class BitLengthNode extends PythonUnaryBuiltinNode {
+        public abstract int execute(Object argument);
+
         @Specialization
         static int bitLength(int argument) {
             return Integer.SIZE - Integer.numberOfLeadingZeros(Math.abs(argument));
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/StringBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/StringBuiltins.java
@@ -75,7 +75,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.oracle.truffle.api.dsl.Fallback;
 import org.graalvm.shadowed.com.ibm.icu.lang.UCharacter;
 import org.graalvm.shadowed.com.ibm.icu.lang.UProperty;
 import org.graalvm.shadowed.com.ibm.icu.text.CaseMap;
@@ -179,6 +178,7 @@
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Cached.Exclusive;
 import com.oracle.truffle.api.dsl.Cached.Shared;
+import com.oracle.truffle.api.dsl.Fallback;
 import com.oracle.truffle.api.dsl.GenerateCached;
 import com.oracle.truffle.api.dsl.GenerateInline;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
@@ -263,7 +263,7 @@ private static TruffleString formatString(Node raisingNode, Spec spec, String st
         private static Spec getAndValidateSpec(Node inliningTarget, TruffleString formatString, PRaiseNode.Lazy raiseNode) {
             Spec spec = InternalFormat.fromText(formatString, 's', '<', inliningTarget);
             if (Spec.specified(spec.type) && spec.type != 's') {
-                throw raiseNode.get(inliningTarget).raise(TypeError, ErrorMessages.UNKNOWN_FORMAT_CODE, spec.type, "str");
+                throw raiseNode.get(inliningTarget).raise(ValueError, ErrorMessages.UNKNOWN_FORMAT_CODE, spec.type, "str");
             }
             if (Spec.specified(spec.sign)) {
                 if (spec.sign == ' ') {
diff --git a/graalpython/lib-graalpython/patches/metadata.toml b/graalpython/lib-graalpython/patches/metadata.toml
@@ -260,9 +260,6 @@ version = '== 3.7.0'
 patch = 'matplotlib-3.7.0.patch'
 license = 'PSF-2.0'
 
-[[meson.rules]]
-version = '== 1.2.2'
-
 [[meson-python.rules]]
 # Upstreamed
 version = '<= 0.16.0'
@@ -461,6 +458,10 @@ version = '== 2.10.1'
 patch = 'pydantic-core-2.10.1.patch'
 license = 'MIT'
 
+[[pydantic-core.rules]]
+version = '== 2.26.0'
+patch = 'pydantic-core-2.26.0.patch'
+license = 'MIT'
 
 [[PyDriller.rules]]
 patch = 'PyDriller.patch'
diff --git a/graalpython/lib-graalpython/patches/pydantic-core-2.26.0.patch b/graalpython/lib-graalpython/patches/pydantic-core-2.26.0.patch
@@ -0,0 +1,165 @@
+diff --git a/Cargo.toml b/Cargo.toml
+index 625e474..e3ad221 100644
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -29,7 +29,7 @@ rust-version = "1.75"
+ [dependencies]
+ # TODO it would be very nice to remove the "py-clone" feature as it can panic,
+ # but needs a bit of work to make sure it's not used in the codebase
+-pyo3 = { version = "0.22.5", features = ["generate-import-lib", "num-bigint", "py-clone"] }
++pyo3 = { version = "0.22.5", features = ["generate-import-lib", "num-bigint", "py-clone"], path = "patched_crates/pyo3" }
+ regex = "1.11.1"
+ strum = { version = "0.26.3", features = ["derive"] }
+ strum_macros = "0.26.4"
+@@ -46,7 +46,7 @@ base64 = "0.22.1"
+ num-bigint = "0.4.6"
+ python3-dll-a = "0.2.10"
+ uuid = "1.11.0"
+-jiter = { version = "0.7", features = ["python"] }
++jiter = { version = "0.7", features = ["python"], path = "patched_crates/jiter/crates/jiter" }
+ hex = "0.4.3"
+ 
+ [lib]
+@@ -74,12 +74,12 @@ debug = true
+ strip = false
+ 
+ [dev-dependencies]
+-pyo3 = { version = "0.22.5", features = ["auto-initialize"] }
++pyo3 = { version = "0.22.5", features = ["auto-initialize"], path = "patched_crates/pyo3" }
+ 
+ [build-dependencies]
+ version_check = "0.9.5"
+ # used where logic has to be version/distribution specific, e.g. pypy
+-pyo3-build-config = { version = "0.22.0" }
++pyo3-build-config = { version = "0.22.0", path = "patched_crates/pyo3/pyo3-build-config" }
+ 
+ [lints.clippy]
+ dbg_macro = "warn"
+diff --git a/build_backend.py b/build_backend.py
+new file mode 100644
+index 0000000..4576d33
+--- /dev/null
++++ b/build_backend.py
+@@ -0,0 +1,23 @@
++import maturin
++import os
++import shutil
++import subprocess
++
++build_sdist = maturin.build_sdist
++
++def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
++    shutil.rmtree('patched_crates', ignore_errors=True)
++    os.makedirs('patched_crates')
++    subprocess.check_call([
++        'git', 'clone', 'https://github.com/PyO3/pyo3.git',
++        'patched_crates/pyo3',
++        '--branch', 'v0.22.5',
++    ])
++    subprocess.check_call(['patch', '-p1', '-d', 'patched_crates/pyo3', '-i', '../../patches/pyo3.patch'])
++    subprocess.check_call([
++        'git', 'clone', 'https://github.com/pydantic/jiter.git',
++        'patched_crates/jiter',
++        '--branch', 'v0.7.0',
++    ])
++    subprocess.check_call(['patch', '-p1', '-d', 'patched_crates/jiter', '-i', '../../patches/jiter.patch'])
++    return maturin.build_wheel(wheel_directory, config_settings, metadata_directory)
+diff --git a/patches/jiter.patch b/patches/jiter.patch
+new file mode 100644
+index 0000000..ce3c408
+--- /dev/null
++++ b/patches/jiter.patch
+@@ -0,0 +1,12 @@
++diff --git a/Cargo.toml b/Cargo.toml
++index 122e111..9ec9e6d 100644
++--- a/Cargo.toml
+++++ b/Cargo.toml
++@@ -29,5 +29,5 @@ inherits = "release"
++ debug = true
++ 
++ [workspace.dependencies]
++-pyo3 = { version = "0.22.0" }
++-pyo3-build-config = { version = "0.22.0" }
+++pyo3 = { version = "0.22.5", path = "../pyo3" }
+++pyo3-build-config = { version = "0.22.5", path = "../pyo3/pyo3-build-config" }
+diff --git a/patches/pyo3.patch b/patches/pyo3.patch
+new file mode 100644
+index 0000000..e488ffb
+--- /dev/null
++++ b/patches/pyo3.patch
+@@ -0,0 +1,31 @@
++diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs
++index 6d17ebc812..1d206f9b18 100644
++--- a/pyo3-ffi/src/cpython/pyerrors.rs
+++++ b/pyo3-ffi/src/cpython/pyerrors.rs
++@@ -6,19 +6,19 @@ use crate::Py_ssize_t;
++ #[derive(Debug)]
++ pub struct PyBaseExceptionObject {
++     pub ob_base: PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub dict: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub args: *mut PyObject,
++-    #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))]
+++    #[cfg(all(Py_3_11, not(PyPy)))]
++     pub notes: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub traceback: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub context: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub cause: *mut PyObject,
++-    #[cfg(not(any(PyPy, GraalPy)))]
+++    #[cfg(not(PyPy))]
++     pub suppress_context: char,
++ }
++ 
+diff --git a/pyproject.toml b/pyproject.toml
+index 9d8e5f0..b43c6c4 100644
+--- a/pyproject.toml
++++ b/pyproject.toml
+@@ -3,7 +3,8 @@ requires = [
+     'maturin>=1,<2',
+     'typing-extensions >=4.6.0,!=4.7.0'
+ ]
+-build-backend = 'maturin'
++backend-path = ['.']
++build-backend = 'build_backend'
+ 
+ [project]
+ name = 'pydantic_core'
+diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs
+index d5e31b9..52289ff 100644
+--- a/src/input/return_enums.rs
++++ b/src/input/return_enums.rs
+@@ -10,7 +10,7 @@ use pyo3::exceptions::PyTypeError;
+ use pyo3::ffi;
+ use pyo3::intern;
+ use pyo3::prelude::*;
+-#[cfg(not(PyPy))]
++#[cfg(all(not(PyPy),not(GraalPy)))]
+ use pyo3::types::PyFunction;
+ use pyo3::types::{PyBytes, PyComplex, PyFloat, PyFrozenSet, PyIterator, PyMapping, PySet, PyString};
+ 
+@@ -321,14 +321,14 @@ pub(crate) fn iterate_attributes<'a, 'py>(
+                 // the PyFunction::is_type_of(attr) catches `staticmethod`, but also any other function,
+                 // I think that's better than including static methods in the yielded attributes,
+                 // if someone really wants fields, they can use an explicit field, or a function to modify input
+-                #[cfg(not(PyPy))]
++                #[cfg(all(not(PyPy),not(GraalPy)))]
+                 if !is_bound && !attr.is_instance_of::<PyFunction>() {
+                     return Some(Ok((name, attr)));
+                 }
+                 // MASSIVE HACK! PyFunction doesn't exist for PyPy,
+                 // is_instance_of::<PyFunction> crashes with a null pointer, hence this hack, see
+                 // https://github.com/pydantic/pydantic-core/pull/161#discussion_r917257635
+-                #[cfg(PyPy)]
++                #[cfg(any(PyPy,GraalPy))]
+                 if !is_bound && attr.get_type().to_string() != "<class 'function'>" {
+                     return Some(Ok((name, attr)));
+                 }

Original file line number	Diff line number	Diff line change
`@@ -483,4 +483,13 @@ static Object convert(Object charPtr, long size, int littleEndian, int signed,`
`483`	`483`	`return fromByteArray.execute(inliningTarget, bytes, littleEndian != 0, signed != 0);`
`484`	`484`	`}`
`485`	`485`	`}`
	`486`	`+`
	`487`	`+ @CApiBuiltin(ret = SIZE_T, args = {PyObject}, call = Direct)`
	`488`	`+ abstract static class _PyLong_NumBits extends CApiUnaryBuiltinNode {`
	`489`	`+ @Specialization`
	`490`	`+ static long numBits(Object obj,`
	`491`	`+ @Cached IntBuiltins.BitLengthNode bitLengthNode) {`
	`492`	`+ return bitLengthNode.execute(obj);`
	`493`	`+ }`
	`494`	`+ }`
`486`	`495`	`}`
Original file line number	Diff line number	Diff line change
`@@ -224,7 +224,7 @@ abstract static class ReprNode extends PythonUnaryBuiltinNode {`
`224`	`224`	`@Specialization`
`225`	`225`	`static TruffleString repr(PGenerator self,`
`226`	`226`	`@Cached SimpleTruffleStringFormatNode simpleTruffleStringFormatNode) {`
`227`		`- return simpleTruffleStringFormatNode.format("<generator object %s at %d>", self.getName(), PythonAbstractObject.objectHashCode(self));`
	`227`	`+ return simpleTruffleStringFormatNode.format("<generator object %s at 0x%d>", self.getQualname(), PythonAbstractObject.objectHashCode(self));`
`228`	`228`	`}`
`229`	`229`	`}`
`230`	`230`