[GR-25013] Update stdlib to CPython 3.8.5

PullRequest: graalpython/1177

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/code.h

@@ -94,19 +94,19 @@ typedef struct {
 #define CO_ITERABLE_COROUTINE   0x0100
 #define CO_ASYNC_GENERATOR      0x0200
 
-/* These are no longer used. */
-#if 0
-#define CO_GENERATOR_ALLOWED    0x1000
-#endif
-#define CO_FUTURE_DIVISION      0x2000
-#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */
-#define CO_FUTURE_WITH_STATEMENT  0x8000
-#define CO_FUTURE_PRINT_FUNCTION  0x10000
-#define CO_FUTURE_UNICODE_LITERALS 0x20000
-
-#define CO_FUTURE_BARRY_AS_BDFL  0x40000
-#define CO_FUTURE_GENERATOR_STOP  0x80000
-#define CO_FUTURE_ANNOTATIONS    0x100000
+/* bpo-39562: These constant values are changed in Python 3.9
+   to prevent collision with compiler flags. CO_FUTURE_ and PyCF_
+   constants must be kept unique. PyCF_ constants can use bits from
+   0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */
+#define CO_FUTURE_DIVISION      0x20000
+#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */
+#define CO_FUTURE_WITH_STATEMENT  0x80000
+#define CO_FUTURE_PRINT_FUNCTION  0x100000
+#define CO_FUTURE_UNICODE_LITERALS 0x200000
+
+#define CO_FUTURE_BARRY_AS_BDFL  0x400000
+#define CO_FUTURE_GENERATOR_STOP  0x800000
+#define CO_FUTURE_ANNOTATIONS    0x1000000
 
 /* This value is found in the co_cell2arg array when the associated cell
    variable does not correspond to an argument. */

graalpython/com.oracle.graal.python.cext/include/compile.h

@@ -24,12 +24,18 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
                    CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
                    CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)
 #define PyCF_MASK_OBSOLETE (CO_NESTED)
+
+/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique.
+   PyCF_ constants can use bits from 0x0100 to 0x10000.
+   CO_FUTURE_ constants use bits starting at 0x20000. */
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
+#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \
+                           PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT)
 
 #ifndef Py_LIMITED_API
 typedef struct {

graalpython/com.oracle.graal.python.cext/include/cpython/pystate.h

@@ -60,6 +60,7 @@ struct _ts {
     struct _ts *next;
     PyInterpreterState *interp;
 
+    /* Borrowed reference to the current frame (it can be NULL) */
     struct _frame *frame;
     int recursion_depth;
     char overflowed; /* The stack has overflowed. Allow 50 more calls

graalpython/com.oracle.graal.python.cext/include/object.h

@@ -34,7 +34,7 @@ of data it contains.  An object's type is fixed when it is created.
 Types themselves are represented as objects; an object contains a
 pointer to the corresponding type object.  The type itself has a type
 pointer pointing to the object representing the type 'type', which
-contains a pointer to itself!).
+contains a pointer to itself!.
 
 Objects do not float around in memory; once allocated an object keeps
 the same size and address.  Objects that must hold variable-size data

graalpython/com.oracle.graal.python.cext/include/patchlevel.h

@@ -23,12 +23,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        8
-#define PY_MICRO_VERSION        2
+#define PY_MICRO_VERSION        5
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.8.2"
+#define PY_VERSION              "3.8.5"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

graalpython/com.oracle.graal.python.cext/modules/_cpython_struct.c

@@ -1291,6 +1291,10 @@ prepare_s(PyStructObject *self)
     size_t ncodes;
 
     fmt = PyBytes_AS_STRING(self->s_format);
+    if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) {
+        PyErr_SetString(StructError, "embedded null character");
+        return -1;
+    }
 
     f = whichtable(&fmt);
 
@@ -2371,6 +2375,9 @@ PyInit__cpython_struct(void)
                        "unknown" float format */
                     if (ptr->format == 'd' || ptr->format == 'f')
                         break;
+                    /* Skip _Bool, semantics are different for standard size */
+                    if (ptr->format == '?')
+                        break;
                     ptr->pack = native->pack;
                     ptr->unpack = native->unpack;
                     break;

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_normalization.txt

@@ -1,2 +1 @@
-*NormalizationTest.test_bug_834676
 *graalpython.lib-python.3.test.test_normalization.NormalizationTest.test_bug_834676

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -119,7 +119,7 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final String NAME = "Python";
     public static final int MAJOR = 3;
     public static final int MINOR = 8;
-    public static final int MICRO = 2;
+    public static final int MICRO = 5;
     public static final String VERSION = MAJOR + "." + MINOR + "." + MICRO;
 
     public static final String MIME_TYPE = "text/x-python";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

@@ -118,6 +118,7 @@ public enum PythonBuiltinClassType implements TruffleObject {
     PRLock("RLock", "_thread"),
     PSemLock("SemLock", "_multiprocessing"),
     PSocket("socket", "_socket"),
+    PJavaSignalHandler("java_signal_handler"),
     PStaticmethod("staticmethod", BuiltinNames.BUILTINS),
     PClassmethod("classmethod", BuiltinNames.BUILTINS),
     PScandirIterator("ScandirIterator", "posix"),

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SignalModuleBuiltins.java

@@ -71,9 +71,12 @@
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.object.HiddenKey;
 
+import sun.misc.SignalHandler;
+
 @CoreFunctions(defineModule = "_signal")
 public class SignalModuleBuiltins extends PythonBuiltins {
-    private static ConcurrentHashMap<Integer, Object> signalHandlers = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<Integer, Object> signalHandlers = new ConcurrentHashMap<>();
+    private static final ConcurrentHashMap<Integer, SignalHandler> defaultSignalHandlers = new ConcurrentHashMap<>();
 
     private static final HiddenKey signalQueueKey = new HiddenKey("signalQueue");
     private final ConcurrentLinkedDeque<SignalTriggerAction> signalQueue = new ConcurrentLinkedDeque<>();
@@ -172,22 +175,34 @@ int alarmOvf(PInt seconds) {
         }
     }
 
+    @TruffleBoundary
+    private static Object handlerToPython(SignalHandler handler, int signum) {
+        if (handler == sun.misc.SignalHandler.SIG_DFL) {
+            return Signals.SIG_DFL;
+        } else if (handler == sun.misc.SignalHandler.SIG_IGN) {
+            return Signals.SIG_IGN;
+        } else if (handler instanceof Signals.PythonSignalHandler) {
+            return signalHandlers.getOrDefault(signum, PNone.NONE);
+        } else {
+            // Save default JVM handlers to be restored later
+            defaultSignalHandlers.put(signum, handler);
+            return Signals.SIG_DFL;
+        }
+    }
+
     @Builtin(name = "getsignal", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class GetSignalNode extends PythonUnaryBuiltinNode {
         @Specialization
         @TruffleBoundary
         Object getsignal(int signum) {
-            int currentSignalHandler = Signals.getCurrentSignalHandler(signum);
-            if (currentSignalHandler == Signals.SIG_UNKNOWN) {
-                if (signalHandlers.containsKey(signum)) {
-                    return signalHandlers.get(signum);
-                } else {
-                    return PNone.NONE;
-                }
-            } else {
-                return currentSignalHandler;
-            }
+            return handlerToPython(Signals.getCurrentSignalHandler(signum), signum);
+        }
+
+        @Specialization(limit = "3")
+        Object getsignal(Object signum,
+                        @CachedLibrary("signum") PythonObjectLibrary lib) {
+            return getsignal(lib.asSize(signum));
         }
     }
 
@@ -218,21 +233,19 @@ Object signalId(@SuppressWarnings("unused") PythonModule self, Object signal, Ob
 
         @TruffleBoundary
         private Object signal(int signum, int id) {
-            Object retval;
+            SignalHandler oldHandler;
             try {
-                retval = Signals.setSignalHandler(signum, id);
-            } catch (IllegalArgumentException e) {
-                throw raise(PythonErrorType.TypeError, ErrorMessages.SIGNAL_MUST_BE_SIGIGN_SIGDFL_OR_CALLABLE_OBJ);
-            }
-            if ((int) retval == Signals.SIG_UNKNOWN) {
-                if (signalHandlers.containsKey(signum)) {
-                    retval = signalHandlers.get(signum);
+                if (id == Signals.SIG_DFL && defaultSignalHandlers.containsKey(signum)) {
+                    oldHandler = Signals.setSignalHandler(signum, defaultSignalHandlers.get(signum));
                 } else {
-                    retval = PNone.NONE;
+                    oldHandler = Signals.setSignalHandler(signum, id);
                 }
+            } catch (IllegalArgumentException e) {
+                throw raise(PythonErrorType.TypeError, ErrorMessages.SIGNAL_MUST_BE_SIGIGN_SIGDFL_OR_CALLABLE_OBJ);
             }
-            signalHandlers.put(signum, id);
-            return retval;
+            Object result = handlerToPython(oldHandler, signum);
+            signalHandlers.remove(signum);
+            return result;
         }
 
         @Specialization(guards = "handlerLib.isCallable(handler)", limit = "1")
@@ -248,25 +261,19 @@ Object signalHandler(PythonModule self, Object signal, Object handler,
         private Object signal(PythonModule self, int signum, Object handler, ReadAttributeFromObjectNode readQueueNode, ReadAttributeFromObjectNode readSemaNode) {
             ConcurrentLinkedDeque<SignalTriggerAction> queue = getQueue(self, readQueueNode);
             Semaphore semaphore = getSemaphore(self, readSemaNode);
-            Object retval;
+            SignalHandler oldHandler;
             SignalTriggerAction signalTrigger = new SignalTriggerAction(handler, signum);
             try {
-                retval = Signals.setSignalHandler(signum, () -> {
+                oldHandler = Signals.setSignalHandler(signum, () -> {
                     queue.add(signalTrigger);
                     semaphore.release();
                 });
             } catch (IllegalArgumentException e) {
                 throw raise(PythonErrorType.ValueError, e);
             }
-            if ((int) retval == Signals.SIG_UNKNOWN) {
-                if (signalHandlers.containsKey(signum)) {
-                    retval = signalHandlers.get(signum);
-                } else {
-                    retval = PNone.NONE;
-                }
-            }
+            Object result = handlerToPython(oldHandler, signum);
             signalHandlers.put(signum, handler);
-            return retval;
+            return result;
         }
 
         @SuppressWarnings("unchecked")
@@ -299,7 +306,6 @@ private static Semaphore getSemaphore(PythonModule self, ReadAttributeFromObject
 final class Signals {
     static final int SIG_DFL = 0;
     static final int SIG_IGN = 1;
-    static final int SIG_UNKNOWN = -1;
     private static final int SIGMAX = 31;
     static final String[] signalNames = new String[SIGMAX + 1];
 
@@ -342,7 +348,7 @@ synchronized static void scheduleAlarm(long seconds) {
         new Thread(new Alarm(seconds)).start();
     }
 
-    private static class PythonSignalHandler implements sun.misc.SignalHandler {
+    static class PythonSignalHandler implements sun.misc.SignalHandler {
         private final Runnable handler;
 
         public PythonSignalHandler(Runnable handler) {
@@ -360,13 +366,17 @@ static String signalNumberToName(int signum) {
     }
 
     @TruffleBoundary
-    synchronized static int setSignalHandler(int signum, Runnable handler) throws IllegalArgumentException {
-        sun.misc.SignalHandler oldH = sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), new PythonSignalHandler(handler));
-        return handlerToInt(oldH);
+    synchronized static SignalHandler setSignalHandler(int signum, Runnable handler) throws IllegalArgumentException {
+        return setSignalHandler(signum, new PythonSignalHandler(handler));
     }
 
     @TruffleBoundary
-    synchronized static int setSignalHandler(int signum, int handler) throws IllegalArgumentException {
+    synchronized static SignalHandler setSignalHandler(int signum, SignalHandler handler) throws IllegalArgumentException {
+        return sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), handler);
+    }
+
+    @TruffleBoundary
+    synchronized static SignalHandler setSignalHandler(int signum, int handler) throws IllegalArgumentException {
         sun.misc.SignalHandler h;
         if (handler == SIG_DFL) {
             h = sun.misc.SignalHandler.SIG_DFL;
@@ -375,34 +385,24 @@ synchronized static int setSignalHandler(int signum, int handler) throws Illegal
         } else {
             throw new IllegalArgumentException();
         }
-        return handlerToInt(sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), h));
+        return sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), h);
     }
 
     @TruffleBoundary
-    synchronized static int getCurrentSignalHandler(int signum) {
+    synchronized static SignalHandler getCurrentSignalHandler(int signum) {
         // To check what the current signal handler, we install default to get the current one
         // and immediately replace it again.
         sun.misc.SignalHandler oldH;
         try {
             oldH = sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), sun.misc.SignalHandler.SIG_DFL);
         } catch (IllegalArgumentException e) {
-            return SIG_DFL;
+            return sun.misc.SignalHandler.SIG_DFL;
         }
         try {
             sun.misc.Signal.handle(new sun.misc.Signal(signalNumberToName(signum)), oldH);
         } catch (IllegalArgumentException e) {
-            return SIG_DFL;
-        }
-        return handlerToInt(oldH);
-    }
-
-    private static int handlerToInt(sun.misc.SignalHandler oldH) {
-        if (oldH == sun.misc.SignalHandler.SIG_DFL) {
-            return SIG_DFL;
-        } else if (oldH == sun.misc.SignalHandler.SIG_IGN) {
-            return SIG_IGN;
-        } else {
-            return SIG_UNKNOWN;
+            return sun.misc.SignalHandler.SIG_DFL;
         }
+        return oldH;
     }
 }