CPython compatible slots infrastructure

* Implemented slots: nb_bool, sq_length, and mp_length, tp_getattr(o), tp_descr_get, tp_descr_set
* Skeletons: tp_richcompare, tp_init
* CPython compatible slots inheritance (fixup_slot_dispatchers, add_operators)
* Skeleton of native HPy slots, but for now HPy slots pretend to be magic methods that
are added to the type after it is created
* Simple slots 'fuzzer' in scripts directory that compares CPython/GraalPy output
* Check slots code conventions and consistency of PBCT slots with the @corefunctions
    * We want the PBCT slots to be initialized in the static ctor to simply bake
    them into the native image even without pre-initialized context. This also
    shifts the responsibility of synchronization to the VM.
* Handles incompatible slot assignment (myType.tp_descrget = otherType.tp_getattro)
    * make TpSlotWrapper subclasses more lenient when checking arguments count

Author: steve-s

graalpython/com.oracle.graal.python.annotations/src/com/oracle/graal/python/annotations/Slot.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Repeatable;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+@Repeatable(Slot.Slots.class)
+public @interface Slot {
+    SlotKind value();
+
+    /**
+     * The slot node either needs frame for execution (e.g., may call Python code) and/or is complex
+     * enough that indirect call to partially evaluated code is preferred over uncached execution
+     * without frame and without setting up indirect call context.
+     * 
+     * The slot call nodes AST inline slot nodes, but if the inline cache overflows or in the
+     * uncached case, they need to either call uncached slot node or do indirect call if
+     * {@code isComplex} is {@code true}.
+     */
+    boolean isComplex() default false;
+
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.TYPE)
+    @interface Slots {
+        Slot[] value();
+    }
+
+    /**
+     * Some slots do not have fixed signature. This annotation must be used for such slots, the
+     * semantics of its fields is the same as of the field with the same name in the
+     * {@code @Builtin} annotation.
+     */
+    @Retention(RetentionPolicy.RUNTIME)
+    @interface SlotSignature {
+        int minNumOfPositionalArgs() default 0;
+
+        boolean takesVarArgs() default false;
+
+        boolean takesVarKeywordArgs() default false;
+
+        String[] parameterNames() default {};
+
+        boolean needsFrame() default false;
+
+        boolean alwaysNeedsCallerFrame() default false;
+
+        String raiseErrorName() default "";
+    }
+
+    enum SlotKind {
+        nb_bool,
+        sq_length,
+        mp_length,
+        tp_descr_get,
+        tp_get_attro,
+        tp_descr_set
+    }
+}

graalpython/com.oracle.graal.python.benchmarks/python/micro/if-generic-non-builtin.py

@@ -0,0 +1,87 @@
+# Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+# Copyright (c) 2017, 2021, Oracle and/or its affiliates.
+# Copyright (c) 2013, Regents of the University of California
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+# conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list of
+# conditions and the following disclaimer in the documentation and/or other materials provided
+# with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+# OF THE POSSIBILITY OF SUCH DAMAGE.
+# micro benchmark: simple for range loop
+
+# This benchmark hits the generic path in PyObjectIsTrueNode, but it is monomorphic in what gets called there
+# On top of that this exercises the fast-paths for attribute lookup, which takes place in Boolish.__bool__
+
+def measure(num, objs, l):
+    r = 0
+    for i in range(num):  # 50000
+        for j in range(l):
+            if objs[j]:
+                r += 1
+
+    return r
+
+
+class Boolish:
+    def __init__(self, b):
+        self.b = b
+    def __bool__(self):
+        return self.b
+
+
+data = [Boolish(False), Boolish(True)] * 20
+
+def __benchmark__(num=50000):
+    measure(num, data, len(data))

graalpython/com.oracle.graal.python.benchmarks/python/micro/if-generic.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# This benchmark hits the generic path in PyObjectIsTrueNode, but it is monomorphic in what gets called there
+
+def measure(num, objs, l):
+    r = 0
+    for i in range(num):  # 50000
+        for j in range(l):
+            if objs[j]:
+                r += 1
+
+    return r
+
+
+data = [frozenset(), frozenset([1,2,3])] * 20
+
+def __benchmark__(num=50000):
+    measure(num, data, len(data))

graalpython/com.oracle.graal.python.benchmarks/python/micro/if-polymorph.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2024, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# This benchmark hits the generic path in PyObjectIsTrueNode with many different types of objects
+
+def measure(num, objs):
+    l = len(objs)
+    r = 0
+    for i in range(num):  # 50000
+        for j in range(l):
+            if objs[j]:
+                r += 1
+
+    return r
+
+
+class Dummy:
+    pass
+
+class Boolish:
+    def __bool__(self):
+        return False
+
+data = [frozenset(), frozenset([1,2,3]), Dummy(), 1234567891234567899199, measure, Boolish()] * 10
+
+def __benchmark__(num=50000):
+    measure(num, data)

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -9260,6 +9260,9 @@ static void add_slot(PyTypeObject* type, char* name, void* meth, int flags, int
 static int type_ready_graalpy_slot_conv(PyTypeObject* cls) {
 #define ADD_SLOT_CONV(__name__, __meth__, __flags__, __signature__) add_slot(cls, (__name__), (__meth__), (__flags__), (__signature__), NULL)
 
+    // TODO: once all slots are converted, we can do one upcall to managed implementation
+    // of add_operators that will use TpSlots#SLOTDEFS and the same algorithm as CPython
+
     /*
      * NOTE: ADD_SLOT_CONV won't overwrite existing attributes, so the order is crucial and must
      * reflect CPython's 'slotdefs' array.
@@ -9273,13 +9276,13 @@ static int type_ready_graalpy_slot_conv(PyTypeObject* cls) {
 
     // NOTE: The slots may be called from managed code, i.e., we need to wrap the functions
     // and convert arguments that should be C primitives.
-    ADD_SLOT_CONV("__getattribute__", cls->tp_getattr, -2, JWRAPPER_GETATTR);
+    // ADD_SLOT_CONV("__getattribute__", cls->tp_getattr, -2, JWRAPPER_GETATTR); tp_getattr does not have wrapper set in slotdefs and hence is ignored in add_operators
     ADD_SLOT_CONV("__setattr__", cls->tp_setattr, -3, JWRAPPER_SETATTR);
     ADD_SLOT_CONV("__repr__", cls->tp_repr, -1, JWRAPPER_REPR);
     ADD_SLOT_CONV("__hash__", cls->tp_hash, -1, JWRAPPER_HASHFUNC);
     ADD_SLOT_CONV("__call__", cls->tp_call, METH_KEYWORDS | METH_VARARGS, JWRAPPER_CALL);
     ADD_SLOT_CONV("__str__", cls->tp_str, -1, JWRAPPER_STR);
-    ADD_SLOT_CONV("__getattribute__", cls->tp_getattro, -2, JWRAPPER_DIRECT);
+    ADD_SLOT_CONV("__getattribute__", cls->tp_getattro, -2, JWRAPPER_BINARYFUNC);
     ADD_SLOT_CONV("__setattr__", cls->tp_setattro, -3, JWRAPPER_SETATTRO);
     ADD_SLOT_CONV("__delattr__", cls->tp_setattro, -3, JWRAPPER_DELATTRO);
     ADD_SLOT_CONV("__clear__", cls->tp_clear, -1, JWRAPPER_INQUIRY);

graalpython/com.oracle.graal.python.processor/src/META-INF/services/javax.annotation.processing.Processor

@@ -1,3 +1,4 @@
 com.oracle.graal.python.processor.ArgumentClinicProcessor
 com.oracle.graal.python.processor.GenerateEnumConstantsProcessor
 com.oracle.graal.python.processor.CApiBuiltinsProcessor
+com.oracle.graal.python.processor.SlotsProcessor

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/CodeWriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -42,7 +42,8 @@
 
 import java.io.IOException;
 import java.io.Writer;
-import java.util.Arrays;
+import java.util.function.Function;
+import java.util.stream.Stream;
 
 public class CodeWriter implements AutoCloseable {
     private final Writer writer;
@@ -84,8 +85,8 @@ public LineBuilder write(String fmt, Object... args) throws IOException {
             return this;
         }
 
-        public LineBuilder writeEach(int[] items, String delimiter, String fmt) throws IOException {
-            String[] strs = Arrays.stream(items).mapToObj(x -> String.format(fmt, x)).toArray(String[]::new);
+        public <T> LineBuilder writeEach(Stream<T> items, String delimiter, Function<T, String> fun) throws IOException {
+            String[] strs = items.map(fun).toArray(String[]::new);
             return write(String.join(delimiter, strs));
         }