Merge branch 'main' into fix-issue-129250

Author: tom-pytel

.github/workflows/reusable-windows.yml

@@ -24,7 +24,7 @@ env:
 
 jobs:
   build:
-    name: 'build and test (${{ inputs.arch }})'
+    name: ${{ inputs.arch == 'arm64' && 'build' || 'build and test' }} (${{ inputs.arch }})
     runs-on: ${{ inputs.os }}
     timeout-minutes: 60
     env:

Include/cpython/longintrepr.h

@@ -76,8 +76,8 @@ typedef long stwodigits; /* signed variant of twodigits */
     - 1: Zero
     - 2: Negative
 
-   The third lowest bit of lv_tag is reserved for an immortality flag, but is
-   not currently used.
+   The third lowest bit of lv_tag is
+   set to 1 for the small ints.
 
    In a normalized number, ob_digit[ndigits-1] (the most significant
    digit) is never zero.  Also, in all cases, for all valid i,

Include/cpython/pyatomic.h

@@ -574,15 +574,15 @@ static inline void _Py_atomic_fence_release(void);
 
 #if _Py_USE_GCC_BUILTIN_ATOMICS
 #  define Py_ATOMIC_GCC_H
-#  include "cpython/pyatomic_gcc.h"
+#  include "pyatomic_gcc.h"
 #  undef Py_ATOMIC_GCC_H
 #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
 #  define Py_ATOMIC_STD_H
-#  include "cpython/pyatomic_std.h"
+#  include "pyatomic_std.h"
 #  undef Py_ATOMIC_STD_H
 #elif defined(_MSC_VER)
 #  define Py_ATOMIC_MSC_H
-#  include "cpython/pyatomic_msc.h"
+#  include "pyatomic_msc.h"
 #  undef Py_ATOMIC_MSC_H
 #else
 #  error "no available pyatomic implementation for this platform/compiler"

Include/cpython/pythread.h

@@ -22,7 +22,7 @@ PyAPI_DATA(const long long) PY_TIMEOUT_MAX;
     */
 #   define NATIVE_TSS_KEY_T     unsigned long
 #elif defined(HAVE_PTHREAD_STUBS)
-#   include "cpython/pthread_stubs.h"
+#   include "pthread_stubs.h"
 #   define NATIVE_TSS_KEY_T     pthread_key_t
 #else
 #   error "Require native threads. See https://bugs.python.org/issue31370"

Include/internal/pycore_frame.h

@@ -159,13 +159,6 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
     // Don't leave a dangling pointer to the old frame when creating generators
     // and coroutines:
     dest->previous = NULL;
-
-#ifdef Py_GIL_DISABLED
-    PyCodeObject *co = _PyFrame_GetCode(dest);
-    for (int i = stacktop; i < co->co_nlocalsplus + co->co_stacksize; i++) {
-        dest->localsplus[i] = PyStackRef_NULL;
-    }
-#endif
 }
 
 #ifdef Py_GIL_DISABLED
@@ -222,16 +215,6 @@ _PyFrame_Initialize(
     for (int i = null_locals_from; i < code->co_nlocalsplus; i++) {
         frame->localsplus[i] = PyStackRef_NULL;
     }
-
-#ifdef Py_GIL_DISABLED
-    // On GIL disabled, we walk the entire stack in GC. Since stacktop
-    // is not always in sync with the real stack pointer, we have
-    // no choice but to traverse the entire stack.
-    // This just makes sure we don't pass the GC invalid stack values.
-    for (int i = code->co_nlocalsplus; i < code->co_nlocalsplus + code->co_stacksize; i++) {
-        frame->localsplus[i] = PyStackRef_NULL;
-    }
-#endif
 }
 
 /* Gets the pointer to the locals array
@@ -405,13 +388,6 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->lltrace = 0;
 #endif
     frame->return_offset = 0;
-
-#ifdef Py_GIL_DISABLED
-    assert(code->co_nlocalsplus == 0);
-    for (int i = 0; i < code->co_stacksize; i++) {
-        frame->localsplus[i] = PyStackRef_NULL;
-    }
-#endif
     return frame;
 }
 

Include/internal/pycore_long.h

@@ -159,13 +159,14 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *);
 
 /* Long value tag bits:
  * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1.
- * 2: Reserved for immortality bit
+ * 2: Set to 1 for the small ints
  * 3+ Unsigned digit count
  */
 #define SIGN_MASK 3
 #define SIGN_ZERO 1
 #define SIGN_NEGATIVE 2
 #define NON_SIZE_BITS 3
+#define IMMORTALITY_BIT_MASK (1 << 2)
 
 /* The functions _PyLong_IsCompact and _PyLong_CompactValue are defined
  * in Include/cpython/longobject.h, since they need to be inline.
@@ -196,7 +197,7 @@ PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *);
 static inline int
 _PyLong_IsNonNegativeCompact(const PyLongObject* op) {
     assert(PyLong_Check(op));
-    return op->long_value.lv_tag <= (1 << NON_SIZE_BITS);
+    return ((op->long_value.lv_tag & ~IMMORTALITY_BIT_MASK) <= (1 << NON_SIZE_BITS));
 }
 
 
@@ -298,7 +299,7 @@ _PyLong_FlipSign(PyLongObject *op) {
         .long_value  = { \
             .lv_tag = TAG_FROM_SIGN_AND_SIZE( \
                 (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \
-                (val) == 0 ? 0 : 1), \
+                (val) == 0 ? 0 : 1) | IMMORTALITY_BIT_MASK, \
             { ((val) >= 0 ? (val) : -(val)) }, \
         } \
     }

Lib/subprocess.py

@@ -43,10 +43,8 @@
 import builtins
 import errno
 import io
-import locale
 import os
 import time
-import signal
 import sys
 import threading
 import warnings
@@ -144,6 +142,8 @@ def __init__(self, returncode, cmd, output=None, stderr=None):
 
     def __str__(self):
         if self.returncode and self.returncode < 0:
+            # Lazy import to improve module import time
+            import signal
             try:
                 return "Command '%s' died with %r." % (
                         self.cmd, signal.Signals(-self.returncode))
@@ -381,6 +381,8 @@ def _text_encoding():
     if sys.flags.utf8_mode:
         return "utf-8"
     else:
+        # Lazy import to improve module import time
+        import locale
         return locale.getencoding()
 
 
@@ -1664,6 +1666,9 @@ def send_signal(self, sig):
             # Don't signal a process that we know has already died.
             if self.returncode is not None:
                 return
+
+            # Lazy import to improve module import time
+            import signal
             if sig == signal.SIGTERM:
                 self.terminate()
             elif sig == signal.CTRL_C_EVENT:
@@ -1765,6 +1770,9 @@ def _posix_spawn(self, args, executable, env, restore_signals, close_fds,
             """Execute program using os.posix_spawn()."""
             kwargs = {}
             if restore_signals:
+                # Lazy import to improve module import time
+                import signal
+
                 # See _Py_RestoreSignals() in Python/pylifecycle.c
                 sigset = []
                 for signame in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ'):
@@ -2214,9 +2222,13 @@ def send_signal(self, sig):
         def terminate(self):
             """Terminate the process with SIGTERM
             """
+            # Lazy import to improve module import time
+            import signal
             self.send_signal(signal.SIGTERM)
 
         def kill(self):
             """Kill the process with SIGKILL
             """
+            # Lazy import to improve module import time
+            import signal
             self.send_signal(signal.SIGKILL)

Lib/test/test_call.py

@@ -1,6 +1,6 @@
 import unittest
 from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG,
-                          set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow)
+                          set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow, import_helper)
 try:
     import _testcapi
 except ImportError:
@@ -616,9 +616,6 @@ def testfunction_kw(self, *, kw):
     return self
 
 
-ADAPTIVE_WARMUP_DELAY = 2
-
-
 @unittest.skipIf(_testcapi is None, "requires _testcapi")
 class TestPEP590(unittest.TestCase):
 
@@ -802,17 +799,18 @@ def __call__(self, *args):
 
     def test_setvectorcall(self):
         from _testcapi import function_setvectorcall
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
         def f(num): return num + 1
         assert_equal = self.assertEqual
         num = 10
         assert_equal(11, f(num))
         function_setvectorcall(f)
-        # make sure specializer is triggered by running > 50 times
-        for _ in range(10 * ADAPTIVE_WARMUP_DELAY):
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
             assert_equal("overridden", f(num))
 
     def test_setvectorcall_load_attr_specialization_skip(self):
         from _testcapi import function_setvectorcall
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
 
         class X:
             def __getattribute__(self, attr):
@@ -824,11 +822,12 @@ def __getattribute__(self, attr):
         function_setvectorcall(X.__getattribute__)
         # make sure specialization doesn't trigger
         # when vectorcall is overridden
-        for _ in range(ADAPTIVE_WARMUP_DELAY):
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
             assert_equal("overridden", x.a)
 
     def test_setvectorcall_load_attr_specialization_deopt(self):
         from _testcapi import function_setvectorcall
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
 
         class X:
             def __getattribute__(self, attr):
@@ -840,12 +839,12 @@ def get_a(x):
         assert_equal = self.assertEqual
         x = X()
         # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization
-        for _ in range(ADAPTIVE_WARMUP_DELAY):
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
             assert_equal("a", get_a(x))
         function_setvectorcall(X.__getattribute__)
         # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
         # gets deopted due to overridden vectorcall
-        for _ in range(ADAPTIVE_WARMUP_DELAY):
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
             assert_equal("overridden", get_a(x))
 
     @requires_limited_api

Lib/test/test_dis.py

@@ -931,8 +931,6 @@ def extended_arg_quick():
 """% (extended_arg_quick.__code__.co_firstlineno,
       extended_arg_quick.__code__.co_firstlineno + 1,)
 
-ADAPTIVE_WARMUP_DELAY = 2
-
 class DisTestBase(unittest.TestCase):
     "Common utilities for DisTests and TestDisTraceback"
 
@@ -1259,8 +1257,9 @@ def test__try_compile_no_context_exc_on_error(self):
             self.assertIsNone(e.__context__)
 
     @staticmethod
-    def code_quicken(f, times=ADAPTIVE_WARMUP_DELAY):
-        for _ in range(times):
+    def code_quicken(f):
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
+        for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
             f()
 
     @cpython_only
@@ -1306,7 +1305,7 @@ def test_call_specialize(self):
     @requires_specialization
     def test_loop_quicken(self):
         # Loop can trigger a quicken where the loop is located
-        self.code_quicken(loop_test, 4)
+        self.code_quicken(loop_test)
         got = self.get_disassembly(loop_test, adaptive=True)
         jit = import_helper.import_module("_testinternalcapi").jit_enabled()
         expected = dis_loop_test_quickened_code.format("JIT" if jit else "NO_JIT")
@@ -1315,8 +1314,9 @@ def test_loop_quicken(self):
     @cpython_only
     @requires_specialization
     def test_loop_with_conditional_at_end_is_quickened(self):
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
         def for_loop_true(x):
-            for i in range(10):
+            for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
                 if x:
                     pass
 
@@ -1325,7 +1325,7 @@ def for_loop_true(x):
                       self.get_disassembly(for_loop_true, adaptive=True))
 
         def for_loop_false(x):
-            for i in range(10):
+            for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD):
                 if x:
                     pass
 
@@ -1335,7 +1335,7 @@ def for_loop_false(x):
 
         def while_loop():
             i = 0
-            while i < 10:
+            while i < _testinternalcapi.SPECIALIZATION_THRESHOLD:
                 i += 1
 
         while_loop()

Lib/test/test_embed.py

@@ -384,7 +384,9 @@ def test_simple_initialization_api(self):
     def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self):
         # https://github.com/python/cpython/issues/92031
 
-        code = textwrap.dedent("""\
+        _testinternalcapi = import_helper.import_module("_testinternalcapi")
+
+        code = textwrap.dedent(f"""\
             import dis
             import importlib._bootstrap
             import opcode
@@ -411,7 +413,7 @@ def is_specialized(f):
 
             assert not is_specialized(func), "specialized instructions found"
 
-            for i in range(test.test_dis.ADAPTIVE_WARMUP_DELAY):
+            for _ in range({_testinternalcapi.SPECIALIZATION_THRESHOLD}):
                 func(importlib._bootstrap, ["x"], lambda *args: None)
 
             assert is_specialized(func), "no specialized instructions found"