Merge branch 'python-import' into msimacek/GR-25013_python_3.8.5_update

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/code.h

@@ -94,19 +94,19 @@ typedef struct {
 #define CO_ITERABLE_COROUTINE   0x0100
 #define CO_ASYNC_GENERATOR      0x0200
 
-/* These are no longer used. */
-#if 0
-#define CO_GENERATOR_ALLOWED    0x1000
-#endif
-#define CO_FUTURE_DIVISION      0x2000
-#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */
-#define CO_FUTURE_WITH_STATEMENT  0x8000
-#define CO_FUTURE_PRINT_FUNCTION  0x10000
-#define CO_FUTURE_UNICODE_LITERALS 0x20000
-
-#define CO_FUTURE_BARRY_AS_BDFL  0x40000
-#define CO_FUTURE_GENERATOR_STOP  0x80000
-#define CO_FUTURE_ANNOTATIONS    0x100000
+/* bpo-39562: These constant values are changed in Python 3.9
+   to prevent collision with compiler flags. CO_FUTURE_ and PyCF_
+   constants must be kept unique. PyCF_ constants can use bits from
+   0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */
+#define CO_FUTURE_DIVISION      0x20000
+#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */
+#define CO_FUTURE_WITH_STATEMENT  0x80000
+#define CO_FUTURE_PRINT_FUNCTION  0x100000
+#define CO_FUTURE_UNICODE_LITERALS 0x200000
+
+#define CO_FUTURE_BARRY_AS_BDFL  0x400000
+#define CO_FUTURE_GENERATOR_STOP  0x800000
+#define CO_FUTURE_ANNOTATIONS    0x1000000
 
 /* This value is found in the co_cell2arg array when the associated cell
    variable does not correspond to an argument. */

graalpython/com.oracle.graal.python.cext/include/compile.h

@@ -24,12 +24,18 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
                    CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
                    CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)
 #define PyCF_MASK_OBSOLETE (CO_NESTED)
+
+/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique.
+   PyCF_ constants can use bits from 0x0100 to 0x10000.
+   CO_FUTURE_ constants use bits starting at 0x20000. */
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
+#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \
+                           PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT)
 
 #ifndef Py_LIMITED_API
 typedef struct {

graalpython/com.oracle.graal.python.cext/include/cpython/pystate.h

@@ -60,6 +60,7 @@ struct _ts {
     struct _ts *next;
     PyInterpreterState *interp;
 
+    /* Borrowed reference to the current frame (it can be NULL) */
     struct _frame *frame;
     int recursion_depth;
     char overflowed; /* The stack has overflowed. Allow 50 more calls

graalpython/com.oracle.graal.python.cext/include/object.h

@@ -34,7 +34,7 @@ of data it contains.  An object's type is fixed when it is created.
 Types themselves are represented as objects; an object contains a
 pointer to the corresponding type object.  The type itself has a type
 pointer pointing to the object representing the type 'type', which
-contains a pointer to itself!).
+contains a pointer to itself!.
 
 Objects do not float around in memory; once allocated an object keeps
 the same size and address.  Objects that must hold variable-size data

graalpython/com.oracle.graal.python.cext/include/patchlevel.h

@@ -23,12 +23,12 @@
 /*--start constants--*/
 #define PY_MAJOR_VERSION        3
 #define PY_MINOR_VERSION        8
-#define PY_MICRO_VERSION        2
+#define PY_MICRO_VERSION        5
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_FINAL
 #define PY_RELEASE_SERIAL       0
 
 /* Version as a string */
-#define PY_VERSION              "3.8.2"
+#define PY_VERSION              "3.8.5"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

graalpython/com.oracle.graal.python.cext/modules/_cpython_struct.c

@@ -1291,6 +1291,10 @@ prepare_s(PyStructObject *self)
     size_t ncodes;
 
     fmt = PyBytes_AS_STRING(self->s_format);
+    if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) {
+        PyErr_SetString(StructError, "embedded null character");
+        return -1;
+    }
 
     f = whichtable(&fmt);
 
@@ -2371,6 +2375,9 @@ PyInit__cpython_struct(void)
                        "unknown" float format */
                     if (ptr->format == 'd' || ptr->format == 'f')
                         break;
+                    /* Skip _Bool, semantics are different for standard size */
+                    if (ptr->format == '?')
+                        break;
                     ptr->pack = native->pack;
                     ptr->unpack = native->unpack;
                     break;

graalpython/lib-python/3/__future__.py

@@ -68,14 +68,14 @@
 # this module.
 CO_NESTED            = 0x0010   # nested_scopes
 CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
-CO_FUTURE_DIVISION   = 0x2000   # division
-CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
-CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
-CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
-CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
-CO_FUTURE_BARRY_AS_BDFL = 0x40000
-CO_FUTURE_GENERATOR_STOP  = 0x80000 # StopIteration becomes RuntimeError in generators
-CO_FUTURE_ANNOTATIONS     = 0x100000  # annotations become strings at runtime
+CO_FUTURE_DIVISION   = 0x20000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x80000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x100000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x400000
+CO_FUTURE_GENERATOR_STOP  = 0x800000 # StopIteration becomes RuntimeError in generators
+CO_FUTURE_ANNOTATIONS     = 0x1000000  # annotations become strings at runtime
 
 class _Feature:
     def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):

graalpython/lib-python/3/_osx_support.py

@@ -212,7 +212,7 @@ def _remove_universal_flags(_config_vars):
         if cv in _config_vars and cv not in os.environ:
             flags = _config_vars[cv]
             flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII)
-            flags = re.sub('-isysroot [^ \t]*', ' ', flags)
+            flags = re.sub(r'-isysroot\s*\S+', ' ', flags)
             _save_modified_value(_config_vars, cv, flags)
 
     return _config_vars
@@ -288,15 +288,15 @@ def _check_for_unavailable_sdk(_config_vars):
     # to /usr and /System/Library by either a standalone CLT
     # package or the CLT component within Xcode.
     cflags = _config_vars.get('CFLAGS', '')
-    m = re.search(r'-isysroot\s+(\S+)', cflags)
+    m = re.search(r'-isysroot\s*(\S+)', cflags)
     if m is not None:
         sdk = m.group(1)
         if not os.path.exists(sdk):
             for cv in _UNIVERSAL_CONFIG_VARS:
                 # Do not alter a config var explicitly overridden by env var
                 if cv in _config_vars and cv not in os.environ:
                     flags = _config_vars[cv]
-                    flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags)
+                    flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags)
                     _save_modified_value(_config_vars, cv, flags)
 
     return _config_vars
@@ -321,7 +321,7 @@ def compiler_fixup(compiler_so, cc_args):
         stripArch = stripSysroot = True
     else:
         stripArch = '-arch' in cc_args
-        stripSysroot = '-isysroot' in cc_args
+        stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot'))
 
     if stripArch or 'ARCHFLAGS' in os.environ:
         while True:
@@ -339,23 +339,34 @@ def compiler_fixup(compiler_so, cc_args):
 
     if stripSysroot:
         while True:
-            try:
-                index = compiler_so.index('-isysroot')
+            indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
+            if not indices:
+                break
+            index = indices[0]
+            if compiler_so[index] == '-isysroot':
                 # Strip this argument and the next one:
                 del compiler_so[index:index+2]
-            except ValueError:
-                break
+            else:
+                # It's '-isysroot/some/path' in one arg
+                del compiler_so[index:index+1]
 
     # Check if the SDK that is used during compilation actually exists,
     # the universal build requires the usage of a universal SDK and not all
     # users have that installed by default.
     sysroot = None
-    if '-isysroot' in cc_args:
-        idx = cc_args.index('-isysroot')
-        sysroot = cc_args[idx+1]
-    elif '-isysroot' in compiler_so:
-        idx = compiler_so.index('-isysroot')
-        sysroot = compiler_so[idx+1]
+    argvar = cc_args
+    indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')]
+    if not indices:
+        argvar = compiler_so
+        indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')]
+
+    for idx in indices:
+        if argvar[idx] == '-isysroot':
+            sysroot = argvar[idx+1]
+            break
+        else:
+            sysroot = argvar[idx][len('-isysroot'):]
+            break
 
     if sysroot and not os.path.isdir(sysroot):
         from distutils import log

graalpython/lib-python/3/_pydecimal.py

@@ -140,8 +140,11 @@
     # Limits for the C version for compatibility
     'MAX_PREC',  'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
 
-    # C version: compile time choice that enables the thread local context
-    'HAVE_THREADS'
+    # C version: compile time choice that enables the thread local context (deprecated, now always true)
+    'HAVE_THREADS',
+
+    # C version: compile time choice that enables the coroutine local context
+    'HAVE_CONTEXTVAR'
 ]
 
 __xname__ = __name__    # sys.modules lookup (--without-threads)
@@ -172,6 +175,7 @@
 
 # Compatibility with the C version
 HAVE_THREADS = True
+HAVE_CONTEXTVAR = True
 if sys.maxsize == 2**63-1:
     MAX_PREC = 999999999999999999
     MAX_EMAX = 999999999999999999

graalpython/lib-python/3/ast.py

@@ -59,11 +59,12 @@ def literal_eval(node_or_string):
         node_or_string = parse(node_or_string, mode='eval')
     if isinstance(node_or_string, Expression):
         node_or_string = node_or_string.body
+    def _raise_malformed_node(node):
+        raise ValueError(f'malformed node or string: {node!r}')
     def _convert_num(node):
-        if isinstance(node, Constant):
-            if type(node.value) in (int, float, complex):
-                return node.value
-        raise ValueError('malformed node or string: ' + repr(node))
+        if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):
+            _raise_malformed_node(node)
+        return node.value
     def _convert_signed_num(node):
         if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):
             operand = _convert_num(node.operand)
@@ -82,6 +83,8 @@ def _convert(node):
         elif isinstance(node, Set):
             return set(map(_convert, node.elts))
         elif isinstance(node, Dict):
+            if len(node.keys) != len(node.values):
+                _raise_malformed_node(node)
             return dict(zip(map(_convert, node.keys),
                             map(_convert, node.values)))
         elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):
@@ -408,11 +411,11 @@ class NodeTransformer(NodeVisitor):
        class RewriteName(NodeTransformer):
 
            def visit_Name(self, node):
-               return copy_location(Subscript(
+               return Subscript(
                    value=Name(id='data', ctx=Load()),
                    slice=Index(value=Str(s=node.id)),
                    ctx=node.ctx
-               ), node)
+               )
 
     Keep in mind that if the node you're operating on has child nodes you must
     either transform the child nodes yourself or call the :meth:`generic_visit`
@@ -480,6 +483,13 @@ def __instancecheck__(cls, inst):
         return type.__instancecheck__(cls, inst)
 
 def _new(cls, *args, **kwargs):
+    for key in kwargs:
+        if key not in cls._fields:
+            # arbitrary keyword arguments are accepted
+            continue
+        pos = cls._fields.index(key)
+        if pos < len(args):
+            raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}")
     if cls in _const_types:
         return Constant(*args, **kwargs)
     return Constant.__new__(cls, *args, **kwargs)