[GR-17483] [GR-17443] Update to CPython 3.7.4

PullRequest: graalpython/604

Author: timfel

graalpython/com.oracle.graal.python.cext/include/abstract.h

@@ -225,7 +225,7 @@ PyAPI_FUNC(int) _PyObject_HasFastCall(PyObject *callable);
    If nargs is equal to zero, args can be NULL. kwargs can be NULL.
    nargs must be greater or equal to zero.
 
-   Return the result on success. Raise an exception on return NULL on
+   Return the result on success. Raise an exception and return NULL on
    error. */
 PyAPI_FUNC(PyObject *) _PyObject_FastCallDict(
     PyObject *callable,

graalpython/com.oracle.graal.python.cext/include/objimpl.h

@@ -260,7 +260,11 @@ typedef union _gc_head {
         union _gc_head *gc_prev;
         Py_ssize_t gc_refs;
     } gc;
-    double dummy;  /* force worst-case alignment */
+    long double dummy;  /* force worst-case alignment */
+    // malloc returns memory block aligned for any built-in types and
+    // long double is the largest standard C type.
+    // On amd64 linux, long double requires 16 byte alignment.
+    // See bpo-27987 for more discussion.
 } PyGC_Head;
 
 extern PyGC_Head *_PyGC_generation0;

graalpython/com.oracle.graal.python.cext/include/patchlevel.h

@@ -23,12 +23,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        7
-#define PY_MICRO_VERSION        3
+#define PY_MICRO_VERSION        4
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.7.3"
+#define PY_VERSION              "3.7.4"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

graalpython/com.oracle.graal.python.cext/include/pylifecycle.h

@@ -144,7 +144,9 @@ PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetPath(void);
 #ifdef Py_BUILD_CORE
 PyAPI_FUNC(_PyInitError) _PyPathConfig_Init(const _PyCoreConfig *core_config);
-PyAPI_FUNC(PyObject*) _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv);
+PyAPI_FUNC(int) _PyPathConfig_ComputeArgv0(
+    int argc, wchar_t **argv,
+    PyObject **argv0_p);
 PyAPI_FUNC(int) _Py_FindEnvConfigValue(
     FILE *env_file,
     const wchar_t *key,

graalpython/com.oracle.graal.python.cext/include/pymem.h

@@ -60,8 +60,6 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack(
 PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback(
     unsigned int domain,
     uintptr_t ptr);
-
-PyAPI_FUNC(int) _PyMem_IsFreed(void *ptr, size_t size);
 #endif   /* !defined(Py_LIMITED_API) */
 
 

graalpython/com.oracle.graal.python.test/src/tests/cpyext/posixmodule.c

@@ -6123,6 +6123,13 @@ os_getpid_impl(PyObject *module)
 }
 #endif /* HAVE_GETPID */
 
+#ifdef NGROUPS_MAX
+#define MAX_GROUPS NGROUPS_MAX
+#else
+    /* defined to be 16 on Solaris7, so this should be a small number */
+#define MAX_GROUPS 64
+#endif
+
 #ifdef HAVE_GETGROUPLIST
 
 /* AC 3.5: funny apple logic below */
@@ -6135,13 +6142,6 @@ Returns a list of groups to which a user belongs.\n\n\
 static PyObject *
 posix_getgrouplist(PyObject *self, PyObject *args)
 {
-#ifdef NGROUPS_MAX
-#define MAX_GROUPS NGROUPS_MAX
-#else
-    /* defined to be 16 on Solaris7, so this should be a small number */
-#define MAX_GROUPS 64
-#endif
-
     const char *user;
     int i, ngroups;
     PyObject *list;
@@ -6150,7 +6150,16 @@ posix_getgrouplist(PyObject *self, PyObject *args)
 #else
     gid_t *groups, basegid;
 #endif
-    ngroups = MAX_GROUPS;
+
+    /*
+     * NGROUPS_MAX is defined by POSIX.1 as the maximum
+     * number of supplimental groups a users can belong to.
+     * We have to increment it by one because
+     * getgrouplist() returns both the supplemental groups
+     * and the primary group, i.e. all of the groups the
+     * user belongs to.
+     */
+    ngroups = 1 + MAX_GROUPS;
 
 #ifdef __APPLE__
     if (!PyArg_ParseTuple(args, "si:getgrouplist", &user, &basegid))
@@ -6219,13 +6228,6 @@ os_getgroups_impl(PyObject *module)
 /*[clinic end generated code: output=42b0c17758561b56 input=d3f109412e6a155c]*/
 {
     PyObject *result = NULL;
-
-#ifdef NGROUPS_MAX
-#define MAX_GROUPS NGROUPS_MAX
-#else
-    /* defined to be 16 on Solaris7, so this should be a small number */
-#define MAX_GROUPS 64
-#endif
     gid_t grouplist[MAX_GROUPS];
 
     /* On MacOSX getgroups(2) can return more than MAX_GROUPS results

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -107,7 +107,7 @@ public final class PythonLanguage extends TruffleLanguage<PythonContext> {
     public static final String NAME = "Python";
     public static final int MAJOR = 3;
     public static final int MINOR = 7;
-    public static final int MICRO = 3;
+    public static final int MICRO = 4;
     public static final String VERSION = MAJOR + "." + MINOR + "." + MICRO;
 
     public static final String MIME_TYPE = "text/x-python";

graalpython/lib-python/3/_dummy_thread.py

@@ -14,7 +14,7 @@
 # Exports only things specified by thread documentation;
 # skipping obsolete synonyms allocate(), start_new(), exit_thread().
 __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
-           'interrupt_main', 'LockType']
+           'interrupt_main', 'LockType', 'RLock']
 
 # A dummy value
 TIMEOUT_MAX = 2**31
@@ -148,6 +148,38 @@ def __repr__(self):
             hex(id(self))
         )
 
+
+class RLock(LockType):
+    """Dummy implementation of threading._RLock.
+
+    Re-entrant lock can be aquired multiple times and needs to be released
+    just as many times. This dummy implemention does not check wheter the
+    current thread actually owns the lock, but does accounting on the call
+    counts.
+    """
+    def __init__(self):
+        super().__init__()
+        self._levels = 0
+
+    def acquire(self, waitflag=None, timeout=-1):
+        """Aquire the lock, can be called multiple times in succession.
+        """
+        locked = super().acquire(waitflag, timeout)
+        if locked:
+            self._levels += 1
+        return locked
+
+    __enter__ = acquire
+
+    def release(self):
+        """Release needs to be called once for every call to acquire().
+        """
+        if self._levels == 0:
+            raise error
+        if self._levels == 1:
+            super().release()
+        self._levels -= 1
+
 # Used to signal that interrupt_main was called in a "thread"
 _interrupt = False
 # True when not executing in a "thread"

graalpython/lib-python/3/_pyio.py

@@ -287,16 +287,15 @@ class IOBase(metaclass=abc.ABCMeta):
     derived classes can override selectively; the default implementations
     represent a file that cannot be read, written or seeked.
 
-    Even though IOBase does not declare read, readinto, or write because
+    Even though IOBase does not declare read or write because
     their signatures will vary, implementations and clients should
     consider those methods part of the interface. Also, implementations
     may raise UnsupportedOperation when operations they do not support are
     called.
 
     The basic type used for binary data read from or written to a file is
-    bytes. Other bytes-like objects are accepted as method arguments too. In
-    some cases (such as readinto), a writable object is required. Text I/O
-    classes work with str data.
+    bytes. Other bytes-like objects are accepted as method arguments too.
+    Text I/O classes work with str data.
 
     Note that calling any method (even inquiries) on a closed stream is
     undefined. Implementations may raise OSError in this case.
@@ -547,6 +546,11 @@ def readlines(self, hint=None):
         return lines
 
     def writelines(self, lines):
+        """Write a list of lines to the stream.
+
+        Line separators are not added, so it is usual for each of the lines
+        provided to have a line separator at the end.
+        """
         self._checkClosed()
         for line in lines:
             self.write(line)
@@ -835,6 +839,10 @@ class BytesIO(BufferedIOBase):
 
     """Buffered I/O implementation using an in-memory bytes buffer."""
 
+    # Initialize _buffer as soon as possible since it's used by __del__()
+    # which calls close()
+    _buffer = None
+
     def __init__(self, initial_bytes=None):
         buf = bytearray()
         if initial_bytes is not None:
@@ -862,7 +870,8 @@ def getbuffer(self):
         return memoryview(self._buffer)
 
     def close(self):
-        self._buffer.clear()
+        if self._buffer is not None:
+            self._buffer.clear()
         super().close()
 
     def read(self, size=-1):
@@ -1759,8 +1768,7 @@ class TextIOBase(IOBase):
     """Base class for text I/O.
 
     This class provides a character and line based interface to stream
-    I/O. There is no readinto method because Python's character strings
-    are immutable. There is no public constructor.
+    I/O. There is no public constructor.
     """
 
     def read(self, size=-1):
@@ -1933,6 +1941,10 @@ class TextIOWrapper(TextIOBase):
 
     _CHUNK_SIZE = 2048
 
+    # Initialize _buffer as soon as possible since it's used by __del__()
+    # which calls close()
+    _buffer = None
+
     # The write_through argument has no effect here since this
     # implementation always writes through.  The argument is present only
     # so that the signature can match the signature of the C version.

graalpython/lib-python/3/asyncio/base_events.py

@@ -56,11 +56,6 @@
 # before cleanup of cancelled handles is performed.
 _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5
 
-# Exceptions which must not call the exception handler in fatal error
-# methods (_fatal_error())
-_FATAL_ERROR_IGNORE = (BrokenPipeError,
-                       ConnectionResetError, ConnectionAbortedError)
-
 _HAS_IPv6 = hasattr(socket, 'AF_INET6')
 
 # Maximum timeout passed to select to avoid OS limitations
@@ -96,7 +91,7 @@ def _set_reuseport(sock):
                              'SO_REUSEPORT defined but not implemented.')
 
 
-def _ipaddr_info(host, port, family, type, proto):
+def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
     # Try to skip getaddrinfo if "host" is already an IP. Users might have
     # handled name resolution in their own code and pass in resolved IPs.
     if not hasattr(socket, 'inet_pton'):
@@ -145,7 +140,7 @@ def _ipaddr_info(host, port, family, type, proto):
             socket.inet_pton(af, host)
             # The host has already been resolved.
             if _HAS_IPv6 and af == socket.AF_INET6:
-                return af, type, proto, '', (host, port, 0, 0)
+                return af, type, proto, '', (host, port, flowinfo, scopeid)
             else:
                 return af, type, proto, '', (host, port)
         except OSError:
@@ -832,7 +827,7 @@ async def _sock_sendfile_fallback(self, sock, file, offset, count):
                 read = await self.run_in_executor(None, file.readinto, view)
                 if not read:
                     break  # EOF
-                await self.sock_sendall(sock, view)
+                await self.sock_sendall(sock, view[:read])
                 total_sent += read
             return total_sent
         finally:
@@ -1083,11 +1078,11 @@ async def _sendfile_fallback(self, transp, file, offset, count):
                     if blocksize <= 0:
                         return total_sent
                 view = memoryview(buf)[:blocksize]
-                read = file.readinto(view)
+                read = await self.run_in_executor(None, file.readinto, view)
                 if not read:
                     return total_sent  # EOF
                 await proto.drain()
-                transp.write(view)
+                transp.write(view[:read])
                 total_sent += read
         finally:
             if total_sent > 0 and hasattr(file, 'seek'):
@@ -1229,7 +1224,8 @@ async def create_datagram_endpoint(self, protocol_factory,
                     if local_addr:
                         sock.bind(local_address)
                     if remote_addr:
-                        await self.sock_connect(sock, remote_address)
+                        if not allow_broadcast:
+                            await self.sock_connect(sock, remote_address)
                         r_addr = remote_address
                 except OSError as exc:
                     if sock is not None:
@@ -1270,7 +1266,7 @@ async def _ensure_resolved(self, address, *,
                                family=0, type=socket.SOCK_STREAM,
                                proto=0, flags=0, loop):
         host, port = address[:2]
-        info = _ipaddr_info(host, port, family, type, proto)
+        info = _ipaddr_info(host, port, family, type, proto, *address[2:])
         if info is not None:
             # "host" is already a resolved IP.
             return [info]