Merge branch 'main' into range_iter_freelist

Author: eendebakpt

Lib/pydoc.py

@@ -53,6 +53,7 @@ class or function within a module or module in a package.  If the
 #     the current directory is changed with os.chdir(), an incorrect
 #     path will be displayed.
 
+import ast
 import __future__
 import builtins
 import importlib._bootstrap
@@ -384,21 +385,29 @@ def ispackage(path):
     return False
 
 def source_synopsis(file):
-    line = file.readline()
-    while line[:1] == '#' or not line.strip():
-        line = file.readline()
-        if not line: break
-    line = line.strip()
-    if line[:4] == 'r"""': line = line[1:]
-    if line[:3] == '"""':
-        line = line[3:]
-        if line[-1:] == '\\': line = line[:-1]
-        while not line.strip():
-            line = file.readline()
-            if not line: break
-        result = line.split('"""')[0].strip()
-    else: result = None
-    return result
+    """Return the one-line summary of a file object, if present"""
+
+    string = ''
+    try:
+        tokens = tokenize.generate_tokens(file.readline)
+        for tok_type, tok_string, _, _, _ in tokens:
+            if tok_type == tokenize.STRING:
+                string += tok_string
+            elif tok_type == tokenize.NEWLINE:
+                with warnings.catch_warnings():
+                    # Ignore the "invalid escape sequence" warning.
+                    warnings.simplefilter("ignore", SyntaxWarning)
+                    docstring = ast.literal_eval(string)
+                if not isinstance(docstring, str):
+                    return None
+                return docstring.strip().split('\n')[0].strip()
+            elif tok_type == tokenize.OP and tok_string in ('(', ')'):
+                string += tok_string
+            elif tok_type not in (tokenize.COMMENT, tokenize.NL, tokenize.ENCODING):
+                return None
+    except (tokenize.TokenError, UnicodeDecodeError, SyntaxError):
+        return None
+    return None
 
 def synopsis(filename, cache={}):
     """Get the one-line summary out of a module file."""

Lib/test/test_peepholer.py

@@ -1193,5 +1193,56 @@ def get_insts(lno1, lno2, op1, op2):
                         ]
                         self.cfg_optimization_test(insts, expected_insts, consts=list(range(5)))
 
+    def test_list_to_tuple_get_iter(self):
+        # for _ in (*foo, *bar) -> for _ in [*foo, *bar]
+        INTRINSIC_LIST_TO_TUPLE = 6
+        insts = [
+            ("BUILD_LIST", 0, 1),
+            ("LOAD_FAST", 0, 2),
+            ("LIST_EXTEND", 1, 3),
+            ("LOAD_FAST", 1, 4),
+            ("LIST_EXTEND", 1, 5),
+            ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6),
+            ("GET_ITER", None, 7),
+            top := self.Label(),
+            ("FOR_ITER", end := self.Label(), 8),
+            ("STORE_FAST", 2, 9),
+            ("JUMP", top, 10),
+            end,
+            ("END_FOR", None, 11),
+            ("POP_TOP", None, 12),
+            ("LOAD_CONST", 0, 13),
+            ("RETURN_VALUE", None, 14),
+        ]
+        expected_insts = [
+            ("BUILD_LIST", 0, 1),
+            ("LOAD_FAST", 0, 2),
+            ("LIST_EXTEND", 1, 3),
+            ("LOAD_FAST", 1, 4),
+            ("LIST_EXTEND", 1, 5),
+            ("NOP", None, 6),  # ("CALL_INTRINSIC_1", INTRINSIC_LIST_TO_TUPLE, 6),
+            ("GET_ITER", None, 7),
+            top := self.Label(),
+            ("FOR_ITER", end := self.Label(), 8),
+            ("STORE_FAST", 2, 9),
+            ("JUMP", top, 10),
+            end,
+            ("END_FOR", None, 11),
+            ("POP_TOP", None, 12),
+            ("LOAD_CONST", 0, 13),
+            ("RETURN_VALUE", None, 14),
+        ]
+        self.cfg_optimization_test(insts, expected_insts, consts=[None])
+
+    def test_list_to_tuple_get_iter_is_safe(self):
+        a, b = [], []
+        for item in (*(items := [0, 1, 2, 3]),):
+            a.append(item)
+            b.append(items.pop())
+        self.assertEqual(a, [0, 1, 2, 3])
+        self.assertEqual(b, [3, 2, 1, 0])
+        self.assertEqual(items, [])
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_pydoc/test_pydoc.py

@@ -4,6 +4,7 @@
 import contextlib
 import importlib.util
 import inspect
+import io
 import pydoc
 import py_compile
 import keyword
@@ -899,6 +900,82 @@ def test_synopsis(self):
             synopsis = pydoc.synopsis(TESTFN, {})
             self.assertEqual(synopsis, 'line 1: h\xe9')
 
+    def test_source_synopsis(self):
+        def check(source, expected, encoding=None):
+            if isinstance(source, str):
+                source_file = StringIO(source)
+            else:
+                source_file = io.TextIOWrapper(io.BytesIO(source), encoding=encoding)
+            with source_file:
+                result = pydoc.source_synopsis(source_file)
+                self.assertEqual(result, expected)
+
+        check('"""Single line docstring."""',
+              'Single line docstring.')
+        check('"""First line of docstring.\nSecond line.\nThird line."""',
+              'First line of docstring.')
+        check('"""First line of docstring.\\nSecond line.\\nThird line."""',
+              'First line of docstring.')
+        check('"""  Whitespace around docstring.  """',
+              'Whitespace around docstring.')
+        check('import sys\n"""No docstring"""',
+              None)
+        check('  \n"""Docstring after empty line."""',
+              'Docstring after empty line.')
+        check('# Comment\n"""Docstring after comment."""',
+              'Docstring after comment.')
+        check('  # Indented comment\n"""Docstring after comment."""',
+              'Docstring after comment.')
+        check('""""""', # Empty docstring
+              '')
+        check('', # Empty file
+              None)
+        check('"""Embedded\0null byte"""',
+              None)
+        check('"""Embedded null byte"""\0',
+              None)
+        check('"""Café and résumé."""',
+              'Café and résumé.')
+        check("'''Triple single quotes'''",
+              'Triple single quotes')
+        check('"Single double quotes"',
+              'Single double quotes')
+        check("'Single single quotes'",
+              'Single single quotes')
+        check('"""split\\\nline"""',
+              'splitline')
+        check('"""Unrecognized escape \\sequence"""',
+              'Unrecognized escape \\sequence')
+        check('"""Invalid escape seq\\uence"""',
+              None)
+        check('r"""Raw \\stri\\ng"""',
+              'Raw \\stri\\ng')
+        check('b"""Bytes literal"""',
+              None)
+        check('f"""f-string"""',
+              None)
+        check('"""Concatenated""" \\\n"string" \'literals\'',
+              'Concatenatedstringliterals')
+        check('"""String""" + """expression"""',
+              None)
+        check('("""In parentheses""")',
+              'In parentheses')
+        check('("""Multiple lines """\n"""in parentheses""")',
+              'Multiple lines in parentheses')
+        check('()', # tuple
+              None)
+        check(b'# coding: iso-8859-15\n"""\xa4uro sign"""',
+              '€uro sign', encoding='iso-8859-15')
+        check(b'"""\xa4"""', # Decoding error
+              None, encoding='utf-8')
+
+        with tempfile.NamedTemporaryFile(mode='w+', encoding='utf-8') as temp_file:
+            temp_file.write('"""Real file test."""\n')
+            temp_file.flush()
+            temp_file.seek(0)
+            result = pydoc.source_synopsis(temp_file)
+            self.assertEqual(result, "Real file test.")
+
     @requires_docstrings
     def test_synopsis_sourceless(self):
         os = import_helper.import_fresh_module('os')

Lib/test/test_typing.py

@@ -5182,6 +5182,18 @@ class C(B[int]):
                 x = pickle.loads(z)
                 self.assertEqual(s, x)
 
+        # Test ParamSpec args and kwargs
+        global PP
+        PP = ParamSpec('PP')
+        for thing in [PP.args, PP.kwargs]:
+            for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+                with self.subTest(thing=thing, proto=proto):
+                    self.assertEqual(
+                        pickle.loads(pickle.dumps(thing, proto)),
+                        thing,
+                    )
+        del PP
+
     def test_copy_and_deepcopy(self):
         T = TypeVar('T')
         class Node(Generic[T]): ...
@@ -8912,13 +8924,13 @@ class Child1(Base1):
         self.assertEqual(Child1.__mutable_keys__, frozenset({'b'}))
 
         class Base2(TypedDict):
-            a: ReadOnly[int]
+            a: int
 
         class Child2(Base2):
-            b: str
+            b: ReadOnly[str]
 
-        self.assertEqual(Child1.__readonly_keys__, frozenset({'a'}))
-        self.assertEqual(Child1.__mutable_keys__, frozenset({'b'}))
+        self.assertEqual(Child2.__readonly_keys__, frozenset({'b'}))
+        self.assertEqual(Child2.__mutable_keys__, frozenset({'a'}))
 
     def test_cannot_make_mutable_key_readonly(self):
         class Base(TypedDict):
@@ -10129,6 +10141,18 @@ def test_valid_uses(self):
         self.assertEqual(C4.__args__, (Concatenate[int, T, P], T))
         self.assertEqual(C4.__parameters__, (T, P))
 
+    def test_invalid_uses(self):
+        with self.assertRaisesRegex(TypeError, 'Concatenate of no types'):
+            Concatenate[()]
+        with self.assertRaisesRegex(
+            TypeError,
+            (
+                'The last parameter to Concatenate should be a '
+                'ParamSpec variable or ellipsis'
+            ),
+        ):
+            Concatenate[int]
+
     def test_var_substitution(self):
         T = TypeVar('T')
         P = ParamSpec('P')

Misc/NEWS.d/next/Core_and_Builtins/2024-12-15-19-51-54.gh-issue-127970.vdUp-y.rst

@@ -0,0 +1,6 @@
+We now use the location of the ``libpython`` runtime library used in the current
+proccess to determine :data:`sys.base_prefix` on all platforms implementing the
+`dladdr <https://pubs.opengroup.org/onlinepubs/9799919799/functions/dladdr.html>`_
+function defined by the UNIX standard — this includes Linux, Android, macOS,
+iOS, FreeBSD, etc. This was already the case on Windows and macOS Framework
+builds.

Misc/NEWS.d/next/Library/2024-12-17-15-23-40.gh-issue-41872.31LjKY.rst

@@ -0,0 +1,3 @@
+Fix quick extraction of module docstrings from a file in :mod:`pydoc`.
+It now supports docstrings with single quotes, escape sequences,
+raw string literals, and other Python syntax.

Modules/getpath.c

@@ -17,10 +17,13 @@
 #endif
 
 #ifdef __APPLE__
-#  include <dlfcn.h>
 #  include <mach-o/dyld.h>
 #endif
 
+#ifdef HAVE_DLFCN_H
+#  include <dlfcn.h>
+#endif
+
 /* Reference the precompiled getpath.py */
 #include "Python/frozen_modules/getpath.h"
 
@@ -803,36 +806,25 @@ progname_to_dict(PyObject *dict, const char *key)
 static int
 library_to_dict(PyObject *dict, const char *key)
 {
+/* macOS framework builds do not link against a libpython dynamic library, but
+   instead link against a macOS Framework. */
+#if defined(Py_ENABLE_SHARED) || defined(WITH_NEXT_FRAMEWORK)
+
 #ifdef MS_WINDOWS
-#ifdef Py_ENABLE_SHARED
     extern HMODULE PyWin_DLLhModule;
     if (PyWin_DLLhModule) {
         return winmodule_to_dict(dict, key, PyWin_DLLhModule);
     }
 #endif
-#elif defined(WITH_NEXT_FRAMEWORK)
-    static char modPath[MAXPATHLEN + 1];
-    static int modPathInitialized = -1;
-    if (modPathInitialized < 0) {
-        modPathInitialized = 0;
-
-        /* On Mac OS X we have a special case if we're running from a framework.
-           This is because the python home should be set relative to the library,
-           which is in the framework, not relative to the executable, which may
-           be outside of the framework. Except when we're in the build
-           directory... */
-        Dl_info pythonInfo;
-        if (dladdr(&Py_Initialize, &pythonInfo)) {
-            if (pythonInfo.dli_fname) {
-                strncpy(modPath, pythonInfo.dli_fname, MAXPATHLEN);
-                modPathInitialized = 1;
-            }
-        }
-    }
-    if (modPathInitialized > 0) {
-        return decode_to_dict(dict, key, modPath);
+
+#if HAVE_DLADDR
+    Dl_info libpython_info;
+    if (dladdr(&Py_Initialize, &libpython_info) && libpython_info.dli_fname) {
+        return decode_to_dict(dict, key, libpython_info.dli_fname);
     }
 #endif
+#endif
+
     return PyDict_SetItemString(dict, key, Py_None) == 0;
 }