Merge branch 'main' into port-marshal-to-pep757/127936

Author: skirpichev

Doc/library/json.rst

@@ -258,64 +258,89 @@ Basic Usage
       the original one. That is, ``loads(dumps(x)) != x`` if x has non-string
       keys.
 
-.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
+.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \
+                   parse_int=None, parse_constant=None, \
+                   object_pairs_hook=None, **kw)
 
-   Deserialize *fp* (a ``.read()``-supporting :term:`text file` or
-   :term:`binary file` containing a JSON document) to a Python object using
-   this :ref:`conversion table <json-to-py-table>`.
+   Deserialize *fp* to a Python object
+   using the :ref:`JSON-to-Python conversion table <json-to-py-table>`.
 
-   *object_hook* is an optional function that will be called with the result of
-   any object literal decoded (a :class:`dict`).  The return value of
-   *object_hook* will be used instead of the :class:`dict`.  This feature can
-   be used to implement custom decoders (e.g. `JSON-RPC
-   <https://www.jsonrpc.org>`_ class hinting).
+   :param fp:
+      A ``.read()``-supporting :term:`text file` or :term:`binary file`
+      containing the JSON document to be deserialized.
+   :type fp: :term:`file-like object`
 
-   *object_pairs_hook* is an optional function that will be called with the
-   result of any object literal decoded with an ordered list of pairs.  The
-   return value of *object_pairs_hook* will be used instead of the
-   :class:`dict`.  This feature can be used to implement custom decoders.  If
-   *object_hook* is also defined, the *object_pairs_hook* takes priority.
+   :param cls:
+      If set, a custom JSON decoder.
+      Additional keyword arguments to :func:`!load`
+      will be passed to the constructor of *cls*.
+      If ``None`` (the default), :class:`!JSONDecoder` is used.
+   :type cls: a :class:`JSONDecoder` subclass
+
+   :param object_hook:
+      If set, a function that is called with the result of
+      any object literal decoded (a :class:`dict`).
+      The return value of this function will be used
+      instead of the :class:`dict`.
+      This feature can be used to implement custom decoders,
+      for example `JSON-RPC <https://www.jsonrpc.org>`_ class hinting.
+      Default ``None``.
+   :type object_hook: :term:`callable` | None
+
+   :param object_pairs_hook:
+      If set, a function that is called with the result of
+      any object literal decoded with an ordered list of pairs.
+      The return value of this function will be used
+      instead of the :class:`dict`.
+      This feature can be used to implement custom decoders.
+      If *object_hook* is also set, *object_pairs_hook* takes priority.
+      Default ``None``.
+   :type object_pairs_hook: :term:`callable` | None
+
+   :param parse_float:
+      If set, a function that is called with
+      the string of every JSON float to be decoded.
+      If ``None`` (the default), it is equivalent to ``float(num_str)``.
+      This can be used to parse JSON floats into custom datatypes,
+      for example :class:`decimal.Decimal`.
+   :type parse_float: :term:`callable` | None
+
+   :param parse_int:
+      If set, a function that is called with
+      the string of every JSON int to be decoded.
+      If ``None`` (the default), it is equivalent to ``int(num_str)``.
+      This can be used to parse JSON integers into custom datatypes,
+      for example :class:`float`.
+   :type parse_int: :term:`callable` | None
+
+   :param parse_constant:
+      If set, a function that is called with one of the following strings:
+      ``'-Infinity'``, ``'Infinity'``, or ``'NaN'``.
+      This can be used to raise an exception
+      if invalid JSON numbers are encountered.
+      Default ``None``.
+   :type parse_constant: :term:`callable` | None
+
+   :raises JSONDecodeError:
+      When the data being deserialized is not a valid JSON document.
 
    .. versionchanged:: 3.1
-      Added support for *object_pairs_hook*.
 
-   *parse_float* is an optional function that will be called with the string of
-   every JSON float to be decoded.  By default, this is equivalent to
-   ``float(num_str)``.  This can be used to use another datatype or parser for
-   JSON floats (e.g. :class:`decimal.Decimal`).
+      * Added the optional *object_pairs_hook* parameter.
+      * *parse_constant* doesn't get called on 'null', 'true', 'false' anymore.
 
-   *parse_int* is an optional function that will be called with the string of
-   every JSON int to be decoded.  By default, this is equivalent to
-   ``int(num_str)``.  This can be used to use another datatype or parser for
-   JSON integers (e.g. :class:`float`).
+   .. versionchanged:: 3.6
+
+      * All optional parameters are now :ref:`keyword-only <keyword-only_parameter>`.
+      * *fp* can now be a :term:`binary file`.
+        The input encoding should be UTF-8, UTF-16 or UTF-32.
 
    .. versionchanged:: 3.11
       The default *parse_int* of :func:`int` now limits the maximum length of
       the integer string via the interpreter's :ref:`integer string
       conversion length limitation <int_max_str_digits>` to help avoid denial
       of service attacks.
 
-   *parse_constant* is an optional function that will be called with one of the
-   following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This can be
-   used to raise an exception if invalid JSON numbers are encountered.
-
-   .. versionchanged:: 3.1
-      *parse_constant* doesn't get called on 'null', 'true', 'false' anymore.
-
-   To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls``
-   kwarg; otherwise :class:`JSONDecoder` is used.  Additional keyword arguments
-   will be passed to the constructor of the class.
-
-   If the data being deserialized is not a valid JSON document, a
-   :exc:`JSONDecodeError` will be raised.
-
-   .. versionchanged:: 3.6
-      All optional parameters are now :ref:`keyword-only <keyword-only_parameter>`.
-
-   .. versionchanged:: 3.6
-      *fp* can now be a :term:`binary file`. The input encoding should be
-      UTF-8, UTF-16 or UTF-32.
-
 .. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw)
 
    Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray`

Lib/asyncio/base_events.py

@@ -477,7 +477,12 @@ def create_task(self, coro, *, name=None, context=None):
 
             task.set_name(name)
 
-        return task
+        try:
+            return task
+        finally:
+            # gh-128552: prevent a refcycle of
+            # task.exception().__traceback__->BaseEventLoop.create_task->task
+            del task
 
     def set_task_factory(self, factory):
         """Set a task factory that will be used by loop.create_task().

Lib/asyncio/taskgroups.py

@@ -205,7 +205,12 @@ def create_task(self, coro, *, name=None, context=None):
         else:
             self._tasks.add(task)
             task.add_done_callback(self._on_task_done)
-        return task
+        try:
+            return task
+        finally:
+            # gh-128552: prevent a refcycle of
+            # task.exception().__traceback__->TaskGroup.create_task->task
+            del task
 
     # Since Python 3.8 Tasks propagate all exceptions correctly,
     # except for KeyboardInterrupt and SystemExit which are

Lib/test/test_asyncio/test_taskgroups.py

@@ -1,6 +1,7 @@
 # Adapted with permission from the EdgeDB project;
 # license: PSFL.
 
+import weakref
 import sys
 import gc
 import asyncio
@@ -38,7 +39,25 @@ def no_other_refs():
     return [coro]
 
 
-class TestTaskGroup(unittest.IsolatedAsyncioTestCase):
+def set_gc_state(enabled):
+    was_enabled = gc.isenabled()
+    if enabled:
+        gc.enable()
+    else:
+        gc.disable()
+    return was_enabled
+
+
+@contextlib.contextmanager
+def disable_gc():
+    was_enabled = set_gc_state(enabled=False)
+    try:
+        yield
+    finally:
+        set_gc_state(enabled=was_enabled)
+
+
+class BaseTestTaskGroup:
 
     async def test_taskgroup_01(self):
 
@@ -832,15 +851,15 @@ async def test_taskgroup_without_parent_task(self):
         with self.assertRaisesRegex(RuntimeError, "has not been entered"):
             tg.create_task(coro)
 
-    def test_coro_closed_when_tg_closed(self):
+    async def test_coro_closed_when_tg_closed(self):
         async def run_coro_after_tg_closes():
             async with taskgroups.TaskGroup() as tg:
                 pass
             coro = asyncio.sleep(0)
             with self.assertRaisesRegex(RuntimeError, "is finished"):
                 tg.create_task(coro)
-        loop = asyncio.get_event_loop()
-        loop.run_until_complete(run_coro_after_tg_closes())
+
+        await run_coro_after_tg_closes()
 
     async def test_cancelling_level_preserved(self):
         async def raise_after(t, e):
@@ -965,6 +984,30 @@ async def coro_fn():
         self.assertIsInstance(exc, _Done)
         self.assertListEqual(gc.get_referrers(exc), no_other_refs())
 
+
+    async def test_exception_refcycles_parent_task_wr(self):
+        """Test that TaskGroup deletes self._parent_task and create_task() deletes task"""
+        tg = asyncio.TaskGroup()
+        exc = None
+
+        class _Done(Exception):
+            pass
+
+        async def coro_fn():
+            async with tg:
+                raise _Done
+
+        with disable_gc():
+            try:
+                async with asyncio.TaskGroup() as tg2:
+                    task_wr = weakref.ref(tg2.create_task(coro_fn()))
+            except* _Done as excs:
+                exc = excs.exceptions[0].exceptions[0]
+
+        self.assertIsNone(task_wr())
+        self.assertIsInstance(exc, _Done)
+        self.assertListEqual(gc.get_referrers(exc), no_other_refs())
+
     async def test_exception_refcycles_propagate_cancellation_error(self):
         """Test that TaskGroup deletes propagate_cancellation_error"""
         tg = asyncio.TaskGroup()
@@ -998,5 +1041,16 @@ class MyKeyboardInterrupt(KeyboardInterrupt):
         self.assertListEqual(gc.get_referrers(exc), no_other_refs())
 
 
+class TestTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase):
+    loop_factory = asyncio.EventLoop
+
+class TestEagerTaskTaskGroup(BaseTestTaskGroup, unittest.IsolatedAsyncioTestCase):
+    @staticmethod
+    def loop_factory():
+        loop = asyncio.EventLoop()
+        loop.set_task_factory(asyncio.eager_task_factory)
+        return loop
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_capi/test_file.py

@@ -5,6 +5,8 @@
 
 _testcapi = import_helper.import_module('_testcapi')
 
+NULL = None
+
 
 class CAPIFileTest(unittest.TestCase):
     def test_py_fopen(self):
@@ -25,15 +27,22 @@ def test_py_fopen(self):
             os_helper.TESTFN,
             os.fsencode(os_helper.TESTFN),
         ]
-        # TESTFN_UNDECODABLE cannot be used to create a file on macOS/WASI.
+        if os_helper.TESTFN_UNDECODABLE is not None:
+            filenames.append(os_helper.TESTFN_UNDECODABLE)
+            filenames.append(os.fsdecode(os_helper.TESTFN_UNDECODABLE))
         if os_helper.TESTFN_UNENCODABLE is not None:
             filenames.append(os_helper.TESTFN_UNENCODABLE)
         for filename in filenames:
             with self.subTest(filename=filename):
                 try:
                     with open(filename, "wb") as fp:
                         fp.write(source)
-
+                except OSError:
+                    # TESTFN_UNDECODABLE cannot be used to create a file
+                    # on macOS/WASI.
+                    filename = None
+                    continue
+                try:
                     data = _testcapi.py_fopen(filename, "rb")
                     self.assertEqual(data, source[:256])
                 finally:
@@ -47,7 +56,14 @@ def test_py_fopen(self):
 
         # non-ASCII mode failing with "Invalid argument"
         with self.assertRaises(OSError):
-            _testcapi.py_fopen(__file__, "\xe9")
+            _testcapi.py_fopen(__file__, b"\xc2\x80")
+        with self.assertRaises(OSError):
+            # \x98 is invalid in cp1250, cp1251, cp1257
+            # \x9d is invalid in cp1252-cp1255, cp1258
+            _testcapi.py_fopen(__file__, b"\xc2\x98\xc2\x9d")
+        # UnicodeDecodeError can come from the audit hook code
+        with self.assertRaises((UnicodeDecodeError, OSError)):
+            _testcapi.py_fopen(__file__, b"\x98\x9d")
 
         # invalid filename type
         for invalid_type in (123, object()):
@@ -60,7 +76,8 @@ def test_py_fopen(self):
                 # On Windows, the file mode is limited to 10 characters
                 _testcapi.py_fopen(__file__, "rt+, ccs=UTF-8")
 
-        # CRASHES py_fopen(__file__, None)
+        # CRASHES _testcapi.py_fopen(NULL, 'rb')
+        # CRASHES _testcapi.py_fopen(__file__, NULL)
 
 
 if __name__ == "__main__":

Lib/test/test_capi/test_opt.py

@@ -1,4 +1,5 @@
 import contextlib
+import itertools
 import sys
 import textwrap
 import unittest
@@ -1511,6 +1512,49 @@ def test_jit_error_pops(self):
         with self.assertRaises(TypeError):
             {item for item in items}
 
+    def test_power_type_depends_on_input_values(self):
+        template = textwrap.dedent("""
+            import _testinternalcapi
+
+            L, R, X, Y = {l}, {r}, {x}, {y}
+
+            def check(actual: complex, expected: complex) -> None:
+                assert actual == expected, (actual, expected)
+                assert type(actual) is type(expected), (actual, expected)
+
+            def f(l: complex, r: complex) -> None:
+                expected_local_local = pow(l, r) + pow(l, r)
+                expected_const_local = pow(L, r) + pow(L, r)
+                expected_local_const = pow(l, R) + pow(l, R)
+                expected_const_const = pow(L, R) + pow(L, R)
+                for _ in range(_testinternalcapi.TIER2_THRESHOLD):
+                    # Narrow types:
+                    l + l, r + r
+                    # The powers produce results, and the addition is unguarded:
+                    check(l ** r + l ** r, expected_local_local)
+                    check(L ** r + L ** r, expected_const_local)
+                    check(l ** R + l ** R, expected_local_const)
+                    check(L ** R + L ** R, expected_const_const)
+
+            # JIT for one pair of values...
+            f(L, R)
+            # ...then run with another:
+            f(X, Y)
+        """)
+        interesting = [
+            (1, 1),  # int ** int -> int
+            (1, -1),  # int ** int -> float
+            (1.0, 1),  # float ** int -> float
+            (1, 1.0),  # int ** float -> float
+            (-1, 0.5),  # int ** float -> complex
+            (1.0, 1.0),  # float ** float -> float
+            (-1.0, 0.5),  # float ** float -> complex
+        ]
+        for (l, r), (x, y) in itertools.product(interesting, repeat=2):
+            s = template.format(l=l, r=r, x=x, y=y)
+            with self.subTest(l=l, r=r, x=x, y=y):
+                script_helper.assert_python_ok("-c", s)
+
 
 def global_identity(x):
     return x