Merge branch 'main' into simplify-no-other-refs

Author: graingert

Doc/deprecations/c-api-pending-removal-in-3.18.rst

@@ -7,7 +7,8 @@ Pending removal in Python 3.18
   * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`.
   * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`.
   * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`.
-  * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`.
+  * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`:
+    use :c:func:`PyLongWriter_Create`.
   * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`.
   * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`.
   * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`.

Doc/library/http.cookies.rst

@@ -142,6 +142,7 @@ Morsel Objects
                     version
                     httponly
                     samesite
+                    partitioned
 
    The attribute :attr:`httponly` specifies that the cookie is only transferred
    in HTTP requests, and is not accessible through JavaScript. This is intended
@@ -151,6 +152,19 @@ Morsel Objects
    send the cookie along with cross-site requests. This helps to mitigate CSRF
    attacks. Valid values for this attribute are "Strict" and "Lax".
 
+   The attribute :attr:`partitioned` indicates to user agents that these
+   cross-site cookies *should* only be available in the same top-level context
+   that the cookie was first set in. For this to be accepted by the user agent,
+   you **must** also set ``Secure``.
+
+   In addition, it is recommended to use the ``__Host`` prefix when setting
+   partitioned cookies to make them bound to the hostname and not the
+   registrable domain. Read
+   `CHIPS (Cookies Having Independent Partitioned State)`_
+   for full details and examples.
+
+   .. _CHIPS (Cookies Having Independent Partitioned State): https://github.com/privacycg/CHIPS/blob/main/README.md
+
    The keys are case-insensitive and their default value is ``''``.
 
    .. versionchanged:: 3.5
@@ -165,6 +179,9 @@ Morsel Objects
    .. versionchanged:: 3.8
       Added support for the :attr:`samesite` attribute.
 
+   .. versionchanged:: 3.14
+      Added support for the :attr:`partitioned` attribute.
+
 
 .. attribute:: Morsel.value
 

Doc/tools/extensions/pyspecific.py

@@ -34,16 +34,6 @@
     Body.enum.converters['lowerroman'] = \
     Body.enum.converters['upperroman'] = lambda x: None
 
-# monkey-patch the productionlist directive to allow hyphens in group names
-# https://github.com/sphinx-doc/sphinx/issues/11854
-from sphinx.domains import std
-
-std.token_re = re.compile(r'`((~?[\w-]*:)?\w+)`')
-
-# backport :no-index:
-PyModule.option_spec['no-index'] = directives.flag
-
-
 # Support for marking up and linking to bugs.python.org issues
 
 def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):

Doc/whatsnew/3.14.rst

@@ -1396,7 +1396,8 @@ Deprecated
   * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`.
   * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`.
   * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`.
-  * :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`.
+  * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`:
+    use :c:func:`PyLongWriter_Create`.
   * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`.
   * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`.
   * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`.

Include/cpython/longintrepr.h

@@ -105,7 +105,7 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t);
 // Return a copy of src.
 PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src);
 
-PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits(
+Py_DEPRECATED(3.14) PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits(
     int negative,
     Py_ssize_t digit_count,
     digit *digits);

Include/internal/pycore_debug_offsets.h

@@ -17,31 +17,32 @@ extern "C" {
 
 // Macros to burn global values in custom sections so out-of-process
 // profilers can locate them easily.
-
-#define GENERATE_DEBUG_SECTION(name, declaration) \
-    _GENERATE_DEBUG_SECTION_WINDOWS(name)         \
-    _GENERATE_DEBUG_SECTION_APPLE(name)           \
-    declaration                                   \
-    _GENERATE_DEBUG_SECTION_LINUX(name)
+#define GENERATE_DEBUG_SECTION(name, declaration)     \
+   _GENERATE_DEBUG_SECTION_WINDOWS(name)            \
+   _GENERATE_DEBUG_SECTION_APPLE(name)              \
+   declaration                                      \
+   _GENERATE_DEBUG_SECTION_LINUX(name)
 
 #if defined(MS_WINDOWS)
 #define _GENERATE_DEBUG_SECTION_WINDOWS(name)                       \
-    _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \
-    __declspec(allocate(Py_STRINGIFY(name)))
+   _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \
+   __declspec(allocate(Py_STRINGIFY(name)))
 #else
 #define _GENERATE_DEBUG_SECTION_WINDOWS(name)
 #endif
 
 #if defined(__APPLE__)
 #define _GENERATE_DEBUG_SECTION_APPLE(name) \
-    __attribute__((section(SEG_DATA "," Py_STRINGIFY(name))))
+   __attribute__((section(SEG_DATA "," Py_STRINGIFY(name))))      \
+   __attribute__((used))
 #else
 #define _GENERATE_DEBUG_SECTION_APPLE(name)
 #endif
 
 #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__))
 #define _GENERATE_DEBUG_SECTION_LINUX(name) \
-    __attribute__((section("." Py_STRINGIFY(name))))
+   __attribute__((section("." Py_STRINGIFY(name))))               \
+   __attribute__((used))
 #else
 #define _GENERATE_DEBUG_SECTION_LINUX(name)
 #endif

Lib/http/cookies.py

@@ -264,11 +264,12 @@ class Morsel(dict):
         "httponly" : "HttpOnly",
         "version"  : "Version",
         "samesite" : "SameSite",
+        "partitioned": "Partitioned",
     }
 
     _reserved_defaults = dict.fromkeys(_reserved, "")
 
-    _flags = {'secure', 'httponly'}
+    _flags = {'secure', 'httponly', 'partitioned'}
 
     def __init__(self):
         # Set defaults

Lib/test/test__colorize.py

@@ -1,80 +1,132 @@
 import contextlib
+import io
 import sys
 import unittest
 import unittest.mock
 import _colorize
-from test.support import force_not_colorized, make_clean_env
+from test.support.os_helper import EnvironmentVarGuard
 
-ORIGINAL_CAN_COLORIZE = _colorize.can_colorize
 
+@contextlib.contextmanager
+def clear_env():
+    with EnvironmentVarGuard() as mock_env:
+        for var in "FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS":
+            mock_env.unset(var)
+        yield mock_env
 
-def setUpModule():
-    _colorize.can_colorize = lambda *args, **kwargs: False
 
-
-def tearDownModule():
-    _colorize.can_colorize = ORIGINAL_CAN_COLORIZE
+def supports_virtual_terminal():
+    if sys.platform == "win32":
+        return unittest.mock.patch("nt._supports_virtual_terminal", return_value=True)
+    else:
+        return contextlib.nullcontext()
 
 
 class TestColorizeFunction(unittest.TestCase):
-    def setUp(self):
-        # Remove PYTHON* environment variables to isolate from local user
-        # settings and simulate running with `-E`. Such variables should be
-        # added to test methods later to patched os.environ.
-        patcher = unittest.mock.patch("os.environ", new=make_clean_env())
-        self.addCleanup(patcher.stop)
-        patcher.start()
-
-    @force_not_colorized
     def test_colorized_detection_checks_for_environment_variables(self):
-        flags = unittest.mock.MagicMock(ignore_environment=False)
+        def check(env, fallback, expected):
+            with (self.subTest(env=env, fallback=fallback),
+                  clear_env() as mock_env):
+                mock_env.update(env)
+                isatty_mock.return_value = fallback
+                stdout_mock.isatty.return_value = fallback
+                self.assertEqual(_colorize.can_colorize(), expected)
+
         with (unittest.mock.patch("os.isatty") as isatty_mock,
               unittest.mock.patch("sys.stdout") as stdout_mock,
-              unittest.mock.patch("sys.stderr") as stderr_mock,
-              unittest.mock.patch("sys.flags", flags),
-              unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE),
-              (unittest.mock.patch("nt._supports_virtual_terminal", return_value=False)
-               if sys.platform == "win32" else
-               contextlib.nullcontext()) as vt_mock):
+              supports_virtual_terminal()):
+            stdout_mock.fileno.return_value = 1
+
+            for fallback in False, True:
+                check({}, fallback, fallback)
+                check({'TERM': 'dumb'}, fallback, False)
+                check({'TERM': 'xterm'}, fallback, fallback)
+                check({'TERM': ''}, fallback, fallback)
+                check({'FORCE_COLOR': '1'}, fallback, True)
+                check({'FORCE_COLOR': '0'}, fallback, True)
+                check({'NO_COLOR': '1'}, fallback, False)
+                check({'NO_COLOR': '0'}, fallback, False)
+
+            check({'TERM': 'dumb', 'FORCE_COLOR': '1'}, False, True)
+            check({'FORCE_COLOR': '1', 'NO_COLOR': '1'}, True, False)
 
+            for ignore_environment in False, True:
+                # Simulate running with or without `-E`.
+                flags = unittest.mock.MagicMock(ignore_environment=ignore_environment)
+                with unittest.mock.patch("sys.flags", flags):
+                    check({'PYTHON_COLORS': '1'}, True, True)
+                    check({'PYTHON_COLORS': '1'}, False, not ignore_environment)
+                    check({'PYTHON_COLORS': '0'}, True, ignore_environment)
+                    check({'PYTHON_COLORS': '0'}, False, False)
+                    for fallback in False, True:
+                        check({'PYTHON_COLORS': 'x'}, fallback, fallback)
+                        check({'PYTHON_COLORS': ''}, fallback, fallback)
+
+                    check({'TERM': 'dumb', 'PYTHON_COLORS': '1'}, False, not ignore_environment)
+                    check({'NO_COLOR': '1', 'PYTHON_COLORS': '1'}, False, not ignore_environment)
+                    check({'FORCE_COLOR': '1', 'PYTHON_COLORS': '0'}, True, ignore_environment)
+
+    @unittest.skipUnless(sys.platform == "win32", "requires Windows")
+    def test_colorized_detection_checks_on_windows(self):
+        with (clear_env(),
+              unittest.mock.patch("os.isatty") as isatty_mock,
+              unittest.mock.patch("sys.stdout") as stdout_mock,
+              supports_virtual_terminal() as vt_mock):
+            stdout_mock.fileno.return_value = 1
             isatty_mock.return_value = True
+            stdout_mock.isatty.return_value = True
+
+            vt_mock.return_value = True
+            self.assertEqual(_colorize.can_colorize(), True)
+            vt_mock.return_value = False
+            self.assertEqual(_colorize.can_colorize(), False)
+            import nt
+            del nt._supports_virtual_terminal
+            self.assertEqual(_colorize.can_colorize(), False)
+
+    def test_colorized_detection_checks_for_std_streams(self):
+        with (clear_env(),
+              unittest.mock.patch("os.isatty") as isatty_mock,
+              unittest.mock.patch("sys.stdout") as stdout_mock,
+              unittest.mock.patch("sys.stderr") as stderr_mock,
+              supports_virtual_terminal()):
             stdout_mock.fileno.return_value = 1
+            stderr_mock.fileno.side_effect = ZeroDivisionError
+            stderr_mock.isatty.side_effect = ZeroDivisionError
+
+            isatty_mock.return_value = True
             stdout_mock.isatty.return_value = True
-            stderr_mock.fileno.return_value = 2
-            stderr_mock.isatty.return_value = True
-            with unittest.mock.patch("os.environ", {'TERM': 'dumb'}):
-                self.assertEqual(_colorize.can_colorize(), False)
-            with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}):
-                self.assertEqual(_colorize.can_colorize(), True)
-            with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}):
-                self.assertEqual(_colorize.can_colorize(), False)
-            with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}):
-                self.assertEqual(_colorize.can_colorize(), False)
-            with unittest.mock.patch("os.environ",
-                                     {'NO_COLOR': '1', "PYTHON_COLORS": '1'}):
-                self.assertEqual(_colorize.can_colorize(), True)
-            with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}):
-                self.assertEqual(_colorize.can_colorize(), True)
-            with unittest.mock.patch("os.environ",
-                                     {'FORCE_COLOR': '1', 'NO_COLOR': '1'}):
-                self.assertEqual(_colorize.can_colorize(), False)
-            with unittest.mock.patch("os.environ",
-                                     {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}):
-                self.assertEqual(_colorize.can_colorize(), False)
-
-            with unittest.mock.patch("os.environ", {}):
-                if sys.platform == "win32":
-                    self.assertEqual(_colorize.can_colorize(), False)
-
-                    vt_mock.return_value = True
-                    self.assertEqual(_colorize.can_colorize(), True)
-                else:
-                    self.assertEqual(_colorize.can_colorize(), True)
+            self.assertEqual(_colorize.can_colorize(), True)
+
+            isatty_mock.return_value = False
+            stdout_mock.isatty.return_value = False
+            self.assertEqual(_colorize.can_colorize(), False)
+
+    def test_colorized_detection_checks_for_file(self):
+        with clear_env(), supports_virtual_terminal():
 
+            with unittest.mock.patch("os.isatty") as isatty_mock:
+                file = unittest.mock.MagicMock()
+                file.fileno.return_value = 1
+                isatty_mock.return_value = True
+                self.assertEqual(_colorize.can_colorize(file=file), True)
                 isatty_mock.return_value = False
-                stdout_mock.isatty.return_value = False
-                stderr_mock.isatty.return_value = False
-                self.assertEqual(_colorize.can_colorize(), False)
+                self.assertEqual(_colorize.can_colorize(file=file), False)
+
+            # No file.fileno.
+            with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError):
+                file = unittest.mock.MagicMock(spec=['isatty'])
+                file.isatty.return_value = True
+                self.assertEqual(_colorize.can_colorize(file=file), False)
+
+            # file.fileno() raises io.UnsupportedOperation.
+            with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError):
+                file = unittest.mock.MagicMock()
+                file.fileno.side_effect = io.UnsupportedOperation
+                file.isatty.return_value = True
+                self.assertEqual(_colorize.can_colorize(file=file), True)
+                file.isatty.return_value = False
+                self.assertEqual(_colorize.can_colorize(file=file), False)
 
 
 if __name__ == "__main__":

Lib/test/test_asyncio/test_free_threading.py

@@ -1,4 +1,5 @@
 import asyncio
+import threading
 import unittest
 from threading import Thread
 from unittest import TestCase
@@ -58,6 +59,42 @@ def runner():
         with threading_helper.start_threads(threads):
             pass
 
+    def test_all_tasks_different_thread(self) -> None:
+        loop = None
+        started = threading.Event()
+        done = threading.Event() # used for main task not finishing early
+        async def coro():
+            await asyncio.Future()
+
+        lock = threading.Lock()
+        tasks = set()
+
+        async def main():
+            nonlocal tasks, loop
+            loop = asyncio.get_running_loop()
+            started.set()
+            for i in range(1000):
+                with lock:
+                    asyncio.create_task(coro())
+                    tasks = self.all_tasks(loop)
+            done.wait()
+
+        runner = threading.Thread(target=lambda: asyncio.run(main()))
+
+        def check():
+            started.wait()
+            with lock:
+                self.assertSetEqual(tasks & self.all_tasks(loop), tasks)
+
+        threads = [threading.Thread(target=check) for _ in range(10)]
+        runner.start()
+
+        with threading_helper.start_threads(threads):
+            pass
+
+        done.set()
+        runner.join()
+
     def test_run_coroutine_threadsafe(self) -> None:
         results = []
 

Lib/test/test_http_cookies.py

@@ -205,6 +205,14 @@ def test_set_secure_httponly_attrs(self):
         self.assertEqual(C.output(),
             'Set-Cookie: Customer="WILE_E_COYOTE"; HttpOnly; Secure')
 
+    def test_set_secure_httponly_partitioned_attrs(self):
+        C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"')
+        C['Customer']['secure'] = True
+        C['Customer']['httponly'] = True
+        C['Customer']['partitioned'] = True
+        self.assertEqual(C.output(),
+            'Set-Cookie: Customer="WILE_E_COYOTE"; HttpOnly; Partitioned; Secure')
+
     def test_samesite_attrs(self):
         samesite_values = ['Strict', 'Lax', 'strict', 'lax']
         for val in samesite_values: