Merge branch 'main' into TracebackException-swallows-cause-context-flasey-exception

Author: YvesDup

Doc/howto/perf_profiling.rst

@@ -254,13 +254,28 @@ files in the current directory which are ELF images for all the JIT trampolines
 that were created by Python.
 
 .. warning::
-    Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take
+    When using ``--call-graph dwarf``, the ``perf`` tool will take
     snapshots of the stack of the process being profiled and save the
-    information in the ``perf.data`` file. By default the size of the stack dump
-    is 8192 bytes but the user can change the size by passing the size after
-    comma like ``--call-graph dwarf,4096``. The size of the stack dump is
-    important because if the size is too small ``perf`` will not be able to
-    unwind the stack and the output will be incomplete. On the other hand, if
-    the size is too big, then ``perf`` won't be able to sample the process as
-    frequently as it would like as the overhead will be higher.
+    information in the ``perf.data`` file. By default, the size of the stack dump
+    is 8192 bytes, but you can change the size by passing it after
+    a comma like ``--call-graph dwarf,16384``.
 
+    The size of the stack dump is important because if the size is too small
+    ``perf`` will not be able to unwind the stack and the output will be
+    incomplete. On the other hand, if the size is too big, then ``perf`` won't
+    be able to sample the process as frequently as it would like as the overhead
+    will be higher.
+
+    The stack size is particularly important when profiling Python code compiled
+    with low optimization levels (like ``-O0``), as these builds tend to have
+    larger stack frames. If you are compiling Python with ``-O0`` and not seeing
+    Python functions in your profiling output, try increasing the stack dump
+    size to 65528 bytes (the maximum)::
+
+        $ perf record -F 9999 -g -k 1 --call-graph dwarf,65528 -o perf.data python -Xperf_jit my_script.py
+
+    Different compilation flags can significantly impact stack sizes:
+
+    - Builds with ``-O0`` typically have much larger stack frames than those with ``-O1`` or higher
+    - Adding optimizations (``-O1``, ``-O2``, etc.) typically reduces stack size
+    - Frame pointers (``-fno-omit-frame-pointer``) generally provide more reliable stack unwinding

Doc/library/dis.rst

@@ -1354,9 +1354,6 @@ iterations of the loop.
    If ``STACK[-1]`` is not ``None``, increments the bytecode counter by *delta*.
    ``STACK[-1]`` is popped.
 
-   This opcode is a pseudo-instruction, replaced in final bytecode by
-   the directed versions (forward/backward).
-
    .. versionadded:: 3.11
 
    .. versionchanged:: 3.12
@@ -1368,9 +1365,6 @@ iterations of the loop.
    If ``STACK[-1]`` is ``None``, increments the bytecode counter by *delta*.
    ``STACK[-1]`` is popped.
 
-   This opcode is a pseudo-instruction, replaced in final bytecode by
-   the directed versions (forward/backward).
-
    .. versionadded:: 3.11
 
    .. versionchanged:: 3.12

Include/internal/pycore_uop_metadata.h

@@ -84,7 +84,7 @@ def get_layout(cls, input_fields, is_struct, base):
         raise ValueError('_align_ must be a non-negative integer')
     elif align == 0:
         # Setting `_align_ = 0` amounts to using the default alignment
-        align == 1
+        align = 1
 
     if base:
         align = max(ctypes.alignment(base), align)

Lib/ctypes/_layout.py

@@ -44,11 +44,19 @@
 doing memory analysis on the Python interpreter, which process tends to
 consume too many resources to run the full regression test non-stop.
 
--S is used to continue running tests after an aborted run.  It will
-maintain the order a standard run (ie, this assumes -r is not used).
+-S is used to resume running tests after an interrupted run.  It will
+maintain the order a standard run (i.e. it assumes -r is not used).
 This is useful after the tests have prematurely stopped for some external
-reason and you want to start running from where you left off rather
-than starting from the beginning.
+reason and you want to resume the run from where you left off rather
+than starting from the beginning. Note: this is different from --prioritize.
+
+--prioritize is used to influence the order of selected tests, such that
+the tests listed as an argument are executed first. This is especially
+useful when combined with -j and -r to pin the longest-running tests
+to start at the beginning of a test run. Pass --prioritize=test_a,test_b
+to make test_a run first, followed by test_b, and then the other tests.
+If test_a wasn't selected for execution by regular means, --prioritize will
+not make it execute.
 
 -f reads the names of tests from the file given as f's argument, one
 or more test names per line.  Whitespace is ignored.  Blank lines and
@@ -236,7 +244,7 @@ def _create_parser():
                        help='wait for user input, e.g., allow a debugger '
                             'to be attached')
     group.add_argument('-S', '--start', metavar='START',
-                       help='the name of the test at which to start.' +
+                       help='resume an interrupted run at the following test.' +
                             more_details)
     group.add_argument('-p', '--python', metavar='PYTHON',
                        help='Command to run Python test subprocesses with.')
@@ -263,6 +271,10 @@ def _create_parser():
     group = parser.add_argument_group('Selecting tests')
     group.add_argument('-r', '--randomize', action='store_true',
                        help='randomize test execution order.' + more_details)
+    group.add_argument('--prioritize', metavar='TEST1,TEST2,...',
+                       action='append', type=priority_list,
+                       help='select these tests first, even if the order is'
+                            ' randomized.' + more_details)
     group.add_argument('-f', '--fromfile', metavar='FILE',
                        help='read names of tests to run from a file.' +
                             more_details)
@@ -406,6 +418,10 @@ def resources_list(string):
     return u
 
 
+def priority_list(string):
+    return string.split(",")
+
+
 def _parse_args(args, **kwargs):
     # Defaults
     ns = Namespace()
@@ -551,4 +567,10 @@ def _parse_args(args, **kwargs):
             print(msg, file=sys.stderr, flush=True)
             sys.exit(2)
 
+    ns.prioritize = [
+        test
+        for test_list in (ns.prioritize or ())
+        for test in test_list
+    ]
+
     return ns

Lib/test/libregrtest/cmdline.py

@@ -142,6 +142,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
             self.random_seed = random.getrandbits(32)
         else:
             self.random_seed = ns.random_seed
+        self.prioritize_tests: tuple[str, ...] = tuple(ns.prioritize)
 
         self.parallel_threads = ns.parallel_threads
 
@@ -237,6 +238,16 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList
         if self.randomize:
             random.shuffle(selected)
 
+        for priority_test in reversed(self.prioritize_tests):
+            try:
+                selected.remove(priority_test)
+            except ValueError:
+                print(f"warning: --prioritize={priority_test} used"
+                        f" but test not actually selected")
+                continue
+            else:
+                selected.insert(0, priority_test)
+
         return (tuple(selected), tests)
 
     @staticmethod

Lib/test/libregrtest/main.py

@@ -1,7 +1,7 @@
 from ctypes import (
     c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof,
     BigEndianStructure, LittleEndianStructure,
-    BigEndianUnion, LittleEndianUnion,
+    BigEndianUnion, LittleEndianUnion, Structure,
 )
 import struct
 import unittest
@@ -69,6 +69,41 @@ class Main(base):
             self.assertEqual(Main.z.offset, 8)
             self.assertEqual(main.z, 7)
 
+    def test_negative_align(self):
+        for base in (Structure, LittleEndianStructure, BigEndianStructure):
+            with (
+                self.subTest(base=base),
+                self.assertRaisesRegex(
+                    ValueError,
+                    '_align_ must be a non-negative integer',
+                )
+            ):
+                class MyStructure(base):
+                    _align_ = -1
+                    _fields_ = []
+
+    def test_zero_align_no_fields(self):
+        for base in (Structure, LittleEndianStructure, BigEndianStructure):
+            with self.subTest(base=base):
+                class MyStructure(base):
+                    _align_ = 0
+                    _fields_ = []
+
+                self.assertEqual(alignment(MyStructure), 1)
+                self.assertEqual(alignment(MyStructure()), 1)
+
+    def test_zero_align_with_fields(self):
+        for base in (Structure, LittleEndianStructure, BigEndianStructure):
+            with self.subTest(base=base):
+                class MyStructure(base):
+                    _align_ = 0
+                    _fields_ = [
+                        ("x", c_ubyte),
+                    ]
+
+                self.assertEqual(alignment(MyStructure), 1)
+                self.assertEqual(alignment(MyStructure()), 1)
+
     def test_oversized_structure(self):
         data = bytearray(b"\0" * 8)
         for base in (LittleEndianStructure, BigEndianStructure):

Lib/test/test_ctypes/test_aligned_structures.py

@@ -3,7 +3,7 @@
 from ctypes import (CDLL, Structure, CFUNCTYPE, pointer,
                     c_void_p, c_char_p, c_wchar_p,
                     c_char, c_wchar, c_int, c_double)
-from test.support import import_helper
+from test.support import import_helper, thread_unsafe
 _ctypes_test = import_helper.import_module("_ctypes_test")
 
 
@@ -21,7 +21,6 @@ def __init__(self, *args, **kw):
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-
 class PickleTest:
     def dumps(self, item):
         return pickle.dumps(item, self.proto)
@@ -39,6 +38,7 @@ def test_simple(self):
             self.assertEqual(memoryview(src).tobytes(),
                                  memoryview(dst).tobytes())
 
+    @thread_unsafe('not thread safe')
     def test_struct(self):
         X.init_called = 0
 

Lib/test/test_ctypes/test_pickling.py

@@ -3,7 +3,7 @@
 import sys
 import unittest
 from test import support
-from test.support import import_helper
+from test.support import import_helper, thread_unsafe
 from test.support import script_helper
 _ctypes_test = import_helper.import_module("_ctypes_test")
 
@@ -13,7 +13,7 @@
 
 dll = ctypes.CDLL(_ctypes_test.__file__)
 
-
+@thread_unsafe('not thread safe')
 class RefcountTestCase(unittest.TestCase):
     @support.refcount_test
     def test_1(self):
@@ -82,7 +82,7 @@ class X(ctypes.Structure):
         gc.collect()
         self.assertEqual(sys.getrefcount(func), orig_refcount)
 
-
+@thread_unsafe('not thread safe')
 class AnotherLeak(unittest.TestCase):
     def test_callback(self):
         proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int)

Lib/test/test_ctypes/test_refcounts.py

@@ -1795,6 +1795,31 @@ def test_gh129093(self):
         self.assertEqual(f'{f'{1!=2=}'=}', "f'{1!=2=}'='1!=2=True'")
         self.assertEqual(f'{f'{1 != 2=}'=}', "f'{1 != 2=}'='1 != 2=True'")
 
+    def test_newlines_in_format_specifiers(self):
+        cases = [
+            """f'{1:d\n}'""",
+            """f'__{
+                1:d
+            }__'""",
+            '''f"{value:.
+               {'2f'}}"''',
+            '''f"{value:
+               {'.2f'}f}"''',
+            '''f"{value:
+                #{'x'}}"''',
+        ]
+        self.assertAllRaise(SyntaxError, "f-string: newlines are not allowed in format specifiers", cases)
+
+        valid_cases = [
+            """f'''__{
+                1:d
+            }__'''""",
+            """f'''{1:d\n}'''""",
+        ]
+
+        for case in valid_cases:
+            compile(case, "<string>", "exec")
+
 
 if __name__ == '__main__':
     unittest.main()