[GR-69732] More natural Java subclassing in Python. Github #470.

docs/user/Native-Extensions.md

@@ -50,5 +50,6 @@ The implementation also relies on `venv` to work, even if you are not using exte
 To support creating multiple GraalPy contexts that access native modules within the same JVM or Native Image, we need to isolate them from each other.
 The current strategy for this is to copy the libraries and modify them such that the dynamic library loader of the operating system will isolate them for us.
 To do this, all GraalPy contexts in the same process (not just those in the same engine!) must set the `python.IsolateNativeModules` option to `true`.
+You should test your applications thoroughly if you want to use this feature, as there are many possiblities for native code to sidestep the library isolation through other process-wide global state.
 
 For more details on this, see [our implementation details](https://github.com/oracle/graalpython/blob/master/docs/contributor/IMPLEMENTATION_DETAILS.md#c-extension-copying).

graalpython/com.oracle.graal.python.test/src/tests/test_interop.py

@@ -720,6 +720,13 @@ def test_super(self):
         super(list, l).remove(0) # ArrayList#remove(int index)
         assert l == [6]
 
+    def test_issubclass_isinstance(self):
+        from java.util import ArrayList, List
+        assert issubclass(ArrayList, List)
+        assert issubclass(ArrayList, ArrayList)
+        assert isinstance(ArrayList(), List)
+        assert isinstance(ArrayList(), ArrayList)
+
     def test_java_array(self):
         import java
         il = java.type("int[]")(20)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

@@ -2420,22 +2420,22 @@ public abstract static class BuildClassNode extends PythonVarargsBuiltinNode {
         public static final TruffleString T_BUILD_JAVA_CLASS = tsLiteral("build_java_class");
 
         @TruffleBoundary
-        private static Object buildJavaClass(Object namespace, TruffleString name, Object base) {
+        private static Object buildJavaClass(Object namespace, TruffleString name, Object base, PKeyword[] keywords) {
             // uncached PythonContext get, since this code path is slow in any case
             Object module = PythonContext.get(null).lookupBuiltinModule(T___GRAALPYTHON__);
             Object buildFunction = PyObjectLookupAttr.executeUncached(module, T_BUILD_JAVA_CLASS);
-            return CallNode.executeUncached(buildFunction, namespace, name, base);
+            return CallNode.executeUncached(buildFunction, new Object[]{namespace, name, base}, keywords);
         }
 
         @InliningCutoff
         private static Object buildJavaClass(VirtualFrame frame, Node inliningTarget, PythonLanguage language, PFunction function, Object[] arguments,
-                        CallDispatchers.FunctionCachedInvokeNode invokeBody,
+                        PKeyword[] keywords, CallDispatchers.FunctionCachedInvokeNode invokeBody,
                         TruffleString name) {
             PDict ns = PFactory.createDict(language, new DynamicObjectStorage(language));
             Object[] args = PArguments.create(0);
             PArguments.setSpecialArgument(args, ns);
             invokeBody.execute(frame, inliningTarget, function, args);
-            return buildJavaClass(ns, name, arguments[1]);
+            return buildJavaClass(ns, name, arguments[1], keywords);
         }
 
         @Specialization
@@ -2478,7 +2478,7 @@ protected Object doItNonFunction(VirtualFrame frame, Object function, Object[] a
 
             if (arguments.length == 2 && env.isHostObject(arguments[1]) && env.asHostObject(arguments[1]) instanceof Class<?>) {
                 // we want to subclass a Java class
-                return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, invokeBody, name);
+                return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, keywords, invokeBody, name);
             }
 
             class InitializeBuildClass {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java

@@ -36,20 +36,25 @@
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.PythonBuiltins;
+import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
+import com.oracle.graal.python.lib.PyObjectGetIter;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.runtime.GilNode;
 import com.oracle.graal.python.runtime.object.PFactory;
+import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.InteropLibrary;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
+import com.oracle.truffle.api.nodes.Node;
 
 /*
  * NOTE: We are not using IndirectCallContext here in this file
@@ -67,10 +72,25 @@ protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFa
     @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false)
     @GenerateNodeFactory
     abstract static class BasesNode extends PythonUnaryBuiltinNode {
-        @Specialization
-        static Object getBases(Object self,
+        @Specialization(limit = "2")
+        static Object getBases(VirtualFrame frame, Object self,
+                        @Bind Node inliningTarget,
+                        @CachedLibrary("self") InteropLibrary lib,
+                        @Cached PyObjectGetIter getIter,
+                        @Cached SequenceStorageNodes.CreateStorageFromIteratorNode createStorageFromIteratorNode,
                         @Bind PythonLanguage language) {
-            return PFactory.createEmptyTuple(language);
+            if (lib.hasMetaParents(self)) {
+                try {
+                    Object parents = lib.getMetaParents(self);
+                    Object iterObj = getIter.execute(frame, inliningTarget, parents);
+                    SequenceStorage storage = createStorageFromIteratorNode.execute(frame, iterObj);
+                    return PFactory.createTuple(language, storage);
+                } catch (UnsupportedMessageException e) {
+                    throw CompilerDirectives.shouldNotReachHere();
+                }
+            } else {
+                return PFactory.createEmptyTuple(language);
+            }
         }
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java

@@ -222,6 +222,7 @@
 import com.oracle.truffle.api.CompilerDirectives.ValueType;
 import com.oracle.truffle.api.HostCompilerDirectives.InliningCutoff;
 import com.oracle.truffle.api.RootCallTarget;
+import com.oracle.truffle.api.TruffleLanguage.Env;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Cached.Exclusive;
@@ -237,6 +238,8 @@
 import com.oracle.truffle.api.frame.Frame;
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.interop.InteropLibrary;
+import com.oracle.truffle.api.interop.UnknownIdentifierException;
+import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.nodes.ControlFlowException;
 import com.oracle.truffle.api.nodes.Node;
@@ -1497,23 +1500,23 @@ static boolean doManaged(PythonManagedClass left, PythonManagedClass right) {
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClassType right) {
+        static boolean doTypeType(PythonBuiltinClassType left, PythonBuiltinClassType right) {
             return left == right;
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClass right) {
+        static boolean doTypeClass(PythonBuiltinClassType left, PythonBuiltinClass right) {
             return left == right.getType();
         }
 
         @Specialization
-        static boolean doManaged(PythonBuiltinClass left, PythonBuiltinClassType right) {
+        static boolean doClassType(PythonBuiltinClass left, PythonBuiltinClassType right) {
             return left.getType() == right;
         }
 
         @Specialization
         @InliningCutoff
-        static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbstractNativeObject right,
+        static boolean doNative(PythonAbstractNativeObject left, PythonAbstractNativeObject right,
                         @CachedLibrary(limit = "1") InteropLibrary lib) {
             if (left == right) {
                 return true;
@@ -1524,6 +1527,42 @@ static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbst
             return lib.isIdentical(left.getPtr(), right.getPtr(), lib);
         }
 
+        @Specialization(guards = {"!isAnyPythonObject(left)", "!isAnyPythonObject(right)"})
+        @InliningCutoff
+        static boolean doOther(Object left, Object right,
+                        @Bind PythonContext context,
+                        @CachedLibrary(limit = "2") InteropLibrary lib) {
+            if (left == right) {
+                return true;
+            }
+            if (lib.isMetaObject(left) && lib.isMetaObject(right)) {
+                // *sigh*... Host classes have split personality with a "static" and a "class"
+                // side, and that affects identity comparisons. And they report their "class" sides
+                // as bases, but importing from Java gives you the "static" side.
+                Env env = context.getEnv();
+                if (env.isHostObject(left) && env.isHostObject(right)) {
+                    // the activation of isMemberReadable and later readMember serves as branch
+                    // profile
+                    boolean leftIsStatic = lib.isMemberReadable(left, "class");
+                    if (leftIsStatic != lib.isMemberReadable(right, "class")) {
+                        try {
+                            if (leftIsStatic) {
+                                left = lib.readMember(left, "class");
+                            } else {
+                                right = lib.readMember(right, "class");
+                            }
+                        } catch (UnsupportedMessageException | UnknownIdentifierException e) {
+                            throw CompilerDirectives.shouldNotReachHere(e);
+                        }
+                    }
+                }
+                if (lib.isIdentical(left, right, lib)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
         @Fallback
         static boolean doOther(@SuppressWarnings("unused") Object left, @SuppressWarnings("unused") Object right) {
             return false;

graalpython/lib-graalpython/__graalpython__.py

@@ -142,7 +142,17 @@ def import_current_as_named_module_with_delegate(module, module_name, delegate_n
 
 
 @builtin
-def build_java_class(module, ns, name, base):
+def build_java_class(module, ns, name, base, old_style=True):
+    if not old_style:
+        return build_new_style_java_class(ns, name, base)
+    import warnings
+    warnings.warn("Subclassing Java classes is going to change "
+                  "to a new instance layout that is hopefully "
+                  "more intuitive. Pass the keyword old_style=False "
+                  "to your class definition to try the new style. "
+                  "The new style will become the default in the next "
+                  "release and the old style will be removed soon after.", DeprecationWarning, 1)
+
     ns['__super__'] = None  # place where store the original java class when instance is created
     ExtenderClass = type("PythonJavaExtenderClass", (object, ), ns)
     HostAdapter = __graalpython__.extend(base)
@@ -158,3 +168,105 @@ def factory (cls, *args):
 
     resultClass.__new__ = classmethod(factory)
     return resultClass
+
+
+@builtin
+def build_new_style_java_class(module, ns, name, base):
+    import polyglot
+
+    # First, generate the Java subclass using the Truffle API. Instances of
+    # this class is what we want to generate.
+    JavaClass = __graalpython__.extend(base)
+
+    # Second, generate the delegate object class. Code calling from Java will
+    # end up delegating methods to an instance of this type and the Java object
+    # will use this delegate instance to manage dynamic attributes.
+    #
+    # The __init__ function would not do what the user thinks, so we take it
+    # out and call it explicitly in the factory below. The `self` passed into
+    # those Python-defined methods is the delegate instance, but that would be
+    # confusing for users. So we wrap all methods to get to the Java instance
+    # and pass that one as `self`.
+    delegate_namespace = dict(**ns)
+    delegate_namespace["__java_init__"] = delegate_namespace.pop("__init__", lambda self, *a, **kw: None)
+
+    def python_to_java_decorator(fun):
+        return lambda self, *args, **kwds: fun(self.__this__, *args, **kwds)
+
+    for n, v in delegate_namespace.items():
+        if type(v) == type(python_to_java_decorator):
+            delegate_namespace[n] = python_to_java_decorator(v)
+    DelegateClass = type(f"PythonDelegateClassFor{base}", (object,), delegate_namespace)
+    DelegateClass.__qualname__ = DelegateClass.__name__
+
+    # Third, generate the class used to inject into the MRO of the generated
+    # Java subclass. Code calling from Python will go through this class for
+    # lookup.
+    #
+    # The `self` passed into those Python-defined methods will be the Java
+    # instance. We add `__getattr__`, `__setattr__`, and `__delattr__`
+    # implementations to look to the Python delegate object when the Java-side
+    # lookup fails. For convenience, we also allow retrieving static fields
+    # from Java.
+    mro_namespace = dict(**ns)
+
+    def java_getattr(self, name):
+        if name == "super":
+            return __graalpython__.super(self)
+        sentinel = object()
+        result = getattr(self.this, name, sentinel)
+        if result is sentinel:
+            return getattr(self.getClass().static, name)
+        else:
+            return result
+
+    mro_namespace['__getattr__'] = java_getattr
+    mro_namespace['__setattr__'] = lambda self, name, value: setattr(self.this, name, value)
+    mro_namespace['__delattr__'] = lambda self, name: delattr(self.this, name)
+
+    @classmethod
+    def factory(cls, *args, **kwds):
+        # create the delegate object
+        delegate = DelegateClass()
+        # create the Java object (remove the class argument and add the delegate instance)
+        java_object = polyglot.__new__(JavaClass, *(args[1:] + (delegate, )))
+        delegate.__this__ = java_object
+        # call the __init__ function on the delegate object now that the Java instance is available
+        delegate.__java_init__(*args[1:], **kwds)
+        return java_object
+
+    mro_namespace['__constructor__'] = factory
+    if '__new__' not in mro_namespace:
+        mro_namespace['__new__'] = classmethod(lambda cls, *args, **kwds: cls.__constructor__(*args, **kwds))
+    MroClass = type(f"PythonMROMixinFor{base}", (object,), mro_namespace)
+    MroClass.__qualname__ = MroClass.__name__
+    polyglot.register_interop_type(JavaClass, MroClass)
+
+    # Finally, generate a factory that implements the factory and type checking
+    # methods and denies inheriting again
+    class FactoryMeta(type):
+        @property
+        def __bases__(self):
+            return (JavaClass,)
+
+        def __instancecheck__(cls, obj):
+            return isinstance(obj, JavaClass)
+
+        def __subclasscheck__(cls, derived):
+            return cls is derived or issubclass(derived, JavaClass)
+
+        def __new__(mcls, name, bases, namespace):
+            if bases:
+                raise NotImplementedError("Grandchildren of Java classes are not supported")
+            return type.__new__(mcls, name, bases, namespace)
+
+    class FactoryClass(metaclass=FactoryMeta):
+        @classmethod
+        def __new__(cls, *args, **kwds):
+            return MroClass.__new__(*args, **kwds)
+
+    FactoryClass.__name__ = ns['__qualname__'].rsplit(".", 1)[-1]
+    FactoryClass.__qualname__ = ns['__qualname__']
+    FactoryClass.__module__ = ns['__module__']
+
+    return FactoryClass