From ba780b3d8b4e4414332c8ced681a545ddc811867 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Sun, 19 Jan 2025 12:27:45 +0100 Subject: [PATCH 1/5] A little more warning about multi-context C extensions --- docs/user/Native-Extensions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/Native-Extensions.md b/docs/user/Native-Extensions.md index 5746d1372a..ecbeb12830 100644 --- a/docs/user/Native-Extensions.md +++ b/docs/user/Native-Extensions.md @@ -50,5 +50,6 @@ The implementation also relies on `venv` to work, even if you are not using exte To support creating multiple GraalPy contexts that access native modules within the same JVM or Native Image, we need to isolate them from each other. The current strategy for this is to copy the libraries and modify them such that the dynamic library loader of the operating system will isolate them for us. To do this, all GraalPy contexts in the same process (not just those in the same engine!) must set the `python.IsolateNativeModules` option to `true`. +You should test your applications thoroughly if you want to use this feature, as there are many possiblities for native code to sidestep the library isolation through other process-wide global state. For more details on this, see [our implementation details](https://github.com/oracle/graalpython/blob/master/docs/contributor/IMPLEMENTATION_DETAILS.md#c-extension-copying). From 020168f05f162fb0fe9aee836889b143c2db2e27 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 20:49:47 +0100 Subject: [PATCH 2/5] return a better bases tuple for foreign types --- .../foreign/ForeignAbstractClassBuiltins.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java index ad90c12136..f4542e8005 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java @@ -36,20 +36,25 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; +import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.interop.InteropLibrary; import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.CachedLibrary; +import com.oracle.truffle.api.nodes.Node; /* * NOTE: We are not using IndirectCallContext here in this file @@ -67,10 +72,25 @@ protected List> getNodeFa @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false) @GenerateNodeFactory abstract static class BasesNode extends PythonUnaryBuiltinNode { - @Specialization - static Object getBases(Object self, + @Specialization(limit = "2") + static Object getBases(VirtualFrame frame, Object self, + @Bind Node inliningTarget, + @CachedLibrary("self") InteropLibrary lib, + @Cached PyObjectGetIter getIter, + @Cached SequenceStorageNodes.CreateStorageFromIteratorNode createStorageFromIteratorNode, @Bind PythonLanguage language) { - return PFactory.createEmptyTuple(language); + if (lib.hasMetaParents(self)) { + try { + Object parents = lib.getMetaParents(self); + Object iterObj = getIter.execute(frame, inliningTarget, parents); + SequenceStorage storage = createStorageFromIteratorNode.execute(frame, iterObj); + return PFactory.createTuple(language, storage); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(); + } + } else { + return PFactory.createEmptyTuple(language); + } } } From c531ef47b7652ebb39401055a6fcd01bf3ef2343 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 20:50:01 +0100 Subject: [PATCH 3/5] fix identity comparison for foreign types --- .../src/tests/test_interop.py | 7 +++ .../builtins/objects/type/TypeNodes.java | 48 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index 63e26b2579..ed0c87c309 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -720,6 +720,13 @@ def test_super(self): super(list, l).remove(0) # ArrayList#remove(int index) assert l == [6] + def test_issubclass_isinstance(self): + from java.util import ArrayList, List + assert issubclass(ArrayList, List) + assert issubclass(ArrayList, ArrayList) + assert isinstance(ArrayList(), List) + assert isinstance(ArrayList(), ArrayList) + def test_java_array(self): import java il = java.type("int[]")(20) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index e8fec852f2..43e8e7db2e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -109,6 +109,7 @@ import com.oracle.graal.python.builtins.modules.WeakRefModuleBuiltinsFactory; import com.oracle.graal.python.builtins.modules.cext.PythonCextTypeBuiltins.GraalPyPrivate_Type_AddMember; import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.PythonAbstractObject; import com.oracle.graal.python.builtins.objects.cell.PCell; import com.oracle.graal.python.builtins.objects.cext.PythonAbstractNativeObject; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; @@ -222,6 +223,7 @@ import com.oracle.truffle.api.CompilerDirectives.ValueType; import com.oracle.truffle.api.HostCompilerDirectives.InliningCutoff; import com.oracle.truffle.api.RootCallTarget; +import com.oracle.truffle.api.TruffleLanguage.Env; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Cached.Exclusive; @@ -237,6 +239,8 @@ import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.ControlFlowException; import com.oracle.truffle.api.nodes.Node; @@ -1497,23 +1501,23 @@ static boolean doManaged(PythonManagedClass left, PythonManagedClass right) { } @Specialization - static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClassType right) { + static boolean doTypeType(PythonBuiltinClassType left, PythonBuiltinClassType right) { return left == right; } @Specialization - static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClass right) { + static boolean doTypeClass(PythonBuiltinClassType left, PythonBuiltinClass right) { return left == right.getType(); } @Specialization - static boolean doManaged(PythonBuiltinClass left, PythonBuiltinClassType right) { + static boolean doClassType(PythonBuiltinClass left, PythonBuiltinClassType right) { return left.getType() == right; } @Specialization @InliningCutoff - static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbstractNativeObject right, + static boolean doNative(PythonAbstractNativeObject left, PythonAbstractNativeObject right, @CachedLibrary(limit = "1") InteropLibrary lib) { if (left == right) { return true; @@ -1524,6 +1528,42 @@ static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbst return lib.isIdentical(left.getPtr(), right.getPtr(), lib); } + @Specialization(guards = {"!isAnyPythonObject(left)", "!isAnyPythonObject(right)"}) + @InliningCutoff + static boolean doOther(Object left, Object right, + @Bind PythonContext context, + @CachedLibrary(limit = "2") InteropLibrary lib) { + if (left == right) { + return true; + } + if (lib.isMetaObject(left) && lib.isMetaObject(right)) { + // *sigh*... Host classes have split personality with a "static" and a "class" + // side, and that affects identity comparisons. And they report their "class" sides + // as bases, but importing from Java gives you the "static" side. + Env env = context.getEnv(); + if (env.isHostObject(left) && env.isHostObject(right)) { + // the activation of isMemberReadable and later readMember serves as branch + // profile + boolean leftIsStatic = lib.isMemberReadable(left, "class"); + if (leftIsStatic != lib.isMemberReadable(right, "class")) { + try { + if (leftIsStatic) { + left = lib.readMember(left, "class"); + } else { + right = lib.readMember(right, "class"); + } + } catch (UnsupportedMessageException | UnknownIdentifierException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + } + if (lib.isIdentical(left, right, lib)) { + return true; + } + } + return false; + } + @Fallback static boolean doOther(@SuppressWarnings("unused") Object left, @SuppressWarnings("unused") Object right) { return false; From 2ea932318b42ccd566d876f66296b9381902c7fb Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 21:01:03 +0100 Subject: [PATCH 4/5] New style subclassing from Java --- .../builtins/modules/BuiltinFunctions.java | 10 +- .../lib-graalpython/__graalpython__.py | 114 +++++++++++++++++- 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java index 139fb6d315..7d2a1c160b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java @@ -2420,22 +2420,22 @@ public abstract static class BuildClassNode extends PythonVarargsBuiltinNode { public static final TruffleString T_BUILD_JAVA_CLASS = tsLiteral("build_java_class"); @TruffleBoundary - private static Object buildJavaClass(Object namespace, TruffleString name, Object base) { + private static Object buildJavaClass(Object namespace, TruffleString name, Object base, PKeyword[] keywords) { // uncached PythonContext get, since this code path is slow in any case Object module = PythonContext.get(null).lookupBuiltinModule(T___GRAALPYTHON__); Object buildFunction = PyObjectLookupAttr.executeUncached(module, T_BUILD_JAVA_CLASS); - return CallNode.executeUncached(buildFunction, namespace, name, base); + return CallNode.executeUncached(buildFunction, new Object[]{namespace, name, base}, keywords); } @InliningCutoff private static Object buildJavaClass(VirtualFrame frame, Node inliningTarget, PythonLanguage language, PFunction function, Object[] arguments, - CallDispatchers.FunctionCachedInvokeNode invokeBody, + PKeyword[] keywords, CallDispatchers.FunctionCachedInvokeNode invokeBody, TruffleString name) { PDict ns = PFactory.createDict(language, new DynamicObjectStorage(language)); Object[] args = PArguments.create(0); PArguments.setSpecialArgument(args, ns); invokeBody.execute(frame, inliningTarget, function, args); - return buildJavaClass(ns, name, arguments[1]); + return buildJavaClass(ns, name, arguments[1], keywords); } @Specialization @@ -2478,7 +2478,7 @@ protected Object doItNonFunction(VirtualFrame frame, Object function, Object[] a if (arguments.length == 2 && env.isHostObject(arguments[1]) && env.asHostObject(arguments[1]) instanceof Class) { // we want to subclass a Java class - return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, invokeBody, name); + return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, keywords, invokeBody, name); } class InitializeBuildClass { diff --git a/graalpython/lib-graalpython/__graalpython__.py b/graalpython/lib-graalpython/__graalpython__.py index 1f9cf47c7f..382788edc6 100644 --- a/graalpython/lib-graalpython/__graalpython__.py +++ b/graalpython/lib-graalpython/__graalpython__.py @@ -142,7 +142,17 @@ def import_current_as_named_module_with_delegate(module, module_name, delegate_n @builtin -def build_java_class(module, ns, name, base): +def build_java_class(module, ns, name, base, old_style=True): + if not old_style: + return build_new_style_java_class(ns, name, base) + import warnings + warnings.warn("Subclassing Java classes is going to change " + "to a new instance layout that is hopefully " + "more intuitive. Pass the keyword old_style=False " + "to your class definition to try the new style. " + "The new style will become the default in the next " + "release and the old style will be removed soon after.", DeprecationWarning, 1) + ns['__super__'] = None # place where store the original java class when instance is created ExtenderClass = type("PythonJavaExtenderClass", (object, ), ns) HostAdapter = __graalpython__.extend(base) @@ -158,3 +168,105 @@ def factory (cls, *args): resultClass.__new__ = classmethod(factory) return resultClass + + +@builtin +def build_new_style_java_class(module, ns, name, base): + import polyglot + + # First, generate the Java subclass using the Truffle API. Instances of + # this class is what we want to generate. + JavaClass = __graalpython__.extend(base) + + # Second, generate the delegate object class. Code calling from Java will + # end up delegating methods to an instance of this type and the Java object + # will use this delegate instance to manage dynamic attributes. + # + # The __init__ function would not do what the user thinks, so we take it + # out and call it explicitly in the factory below. The `self` passed into + # those Python-defined methods is the delegate instance, but that would be + # confusing for users. So we wrap all methods to get to the Java instance + # and pass that one as `self`. + delegate_namespace = dict(**ns) + delegate_namespace["__java_init__"] = delegate_namespace.pop("__init__", lambda self, *a, **kw: None) + + def python_to_java_decorator(fun): + return lambda self, *args, **kwds: fun(self.__this__, *args, **kwds) + + for n, v in delegate_namespace.items(): + if type(v) == type(python_to_java_decorator): + delegate_namespace[n] = python_to_java_decorator(v) + DelegateClass = type(f"PythonDelegateClassFor{base}", (object,), delegate_namespace) + DelegateClass.__qualname__ = DelegateClass.__name__ + + # Third, generate the class used to inject into the MRO of the generated + # Java subclass. Code calling from Python will go through this class for + # lookup. + # + # The `self` passed into those Python-defined methods will be the Java + # instance. We add `__getattr__`, `__setattr__`, and `__delattr__` + # implementations to look to the Python delegate object when the Java-side + # lookup fails. For convenience, we also allow retrieving static fields + # from Java. + mro_namespace = dict(**ns) + + def java_getattr(self, name): + if name == "super": + return __graalpython__.super(self) + sentinel = object() + result = getattr(self.this, name, sentinel) + if result is sentinel: + return getattr(self.getClass().static, name) + else: + return result + + mro_namespace['__getattr__'] = java_getattr + mro_namespace['__setattr__'] = lambda self, name, value: setattr(self.this, name, value) + mro_namespace['__delattr__'] = lambda self, name: delattr(self.this, name) + + @classmethod + def factory(cls, *args, **kwds): + # create the delegate object + delegate = DelegateClass() + # create the Java object (remove the class argument and add the delegate instance) + java_object = polyglot.__new__(JavaClass, *(args[1:] + (delegate, ))) + delegate.__this__ = java_object + # call the __init__ function on the delegate object now that the Java instance is available + delegate.__java_init__(*args[1:], **kwds) + return java_object + + mro_namespace['__constructor__'] = factory + if '__new__' not in mro_namespace: + mro_namespace['__new__'] = classmethod(lambda cls, *args, **kwds: cls.__constructor__(*args, **kwds)) + MroClass = type(f"PythonMROMixinFor{base}", (object,), mro_namespace) + MroClass.__qualname__ = MroClass.__name__ + polyglot.register_interop_type(JavaClass, MroClass) + + # Finally, generate a factory that implements the factory and type checking + # methods and denies inheriting again + class FactoryMeta(type): + @property + def __bases__(self): + return (JavaClass,) + + def __instancecheck__(cls, obj): + return isinstance(obj, JavaClass) + + def __subclasscheck__(cls, derived): + return cls is derived or issubclass(derived, JavaClass) + + def __new__(mcls, name, bases, namespace): + if bases: + raise NotImplementedError("Grandchildren of Java classes are not supported") + return type.__new__(mcls, name, bases, namespace) + + class FactoryClass(metaclass=FactoryMeta): + @classmethod + def __new__(cls, *args, **kwds): + return MroClass.__new__(*args, **kwds) + + FactoryClass.__name__ = ns['__qualname__'].rsplit(".", 1)[-1] + FactoryClass.__qualname__ = ns['__qualname__'] + FactoryClass.__module__ = ns['__module__'] + + return FactoryClass From 7f01228a253a7e8cc02de29e686d507ca331a352 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 16 Sep 2025 10:59:55 +0200 Subject: [PATCH 5/5] Fix style --- .../com/oracle/graal/python/builtins/objects/type/TypeNodes.java | 1 - 1 file changed, 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index 43e8e7db2e..cbe934793d 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -109,7 +109,6 @@ import com.oracle.graal.python.builtins.modules.WeakRefModuleBuiltinsFactory; import com.oracle.graal.python.builtins.modules.cext.PythonCextTypeBuiltins.GraalPyPrivate_Type_AddMember; import com.oracle.graal.python.builtins.objects.PNone; -import com.oracle.graal.python.builtins.objects.PythonAbstractObject; import com.oracle.graal.python.builtins.objects.cell.PCell; import com.oracle.graal.python.builtins.objects.cext.PythonAbstractNativeObject; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass;