Merge branch 'main' into refactor/hashlib/module-methods-135532

Author: picnixz

Doc/whatsnew/3.14.rst

@@ -1757,6 +1757,16 @@ os
   (Contributed by Cody Maloney in :gh:`129205`.)
 
 
+os.path
+-------
+
+* The *strict* parameter to :func:`os.path.realpath` accepts a new value,
+  :data:`os.path.ALLOW_MISSING`.
+  If used, errors other than :exc:`FileNotFoundError` will be re-raised;
+  the resulting path can be missing but it will be free of symlinks.
+  (Contributed by Petr Viktorin for :cve:`2025-4517`.)
+
+
 pathlib
 -------
 
@@ -1945,6 +1955,28 @@ sysconfig
   (Contributed by Xuehai Pan in :gh:`131799`.)
 
 
+tarfile
+-------
+
+* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to
+  avoid path traversal attacks.
+  (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.)
+* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes
+  when a directory was removed or replaced by another kind of file.
+  (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.)
+* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall`
+  now (re-)apply the extraction filter when substituting a link (hard or
+  symbolic) with a copy of another archive member, and when fixing up
+  directory attributes.
+  The former raises a new exception, :exc:`~tarfile.LinkFallbackError`.
+  (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.)
+* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall`
+  no longer extract rejected members when
+  :func:`~tarfile.TarFile.errorlevel` is zero.
+  (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887`
+  and :cve:`2025-4435`.)
+
+
 threading
 ---------
 
@@ -2700,6 +2732,7 @@ New features
   * :c:func:`PyUnicodeWriter_Discard`
   * :c:func:`PyUnicodeWriter_Finish`
   * :c:func:`PyUnicodeWriter_Format`
+  * :c:func:`PyUnicodeWriter_WriteASCII`
   * :c:func:`PyUnicodeWriter_WriteChar`
   * :c:func:`PyUnicodeWriter_WriteRepr`
   * :c:func:`PyUnicodeWriter_WriteStr`
@@ -2976,7 +3009,7 @@ Deprecated
     :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) <PyUnicodeWriter_WriteSubstring>`.
   * :c:func:`!_PyUnicodeWriter_WriteASCIIString`:
     replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with
-    :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) <PyUnicodeWriter_WriteUTF8>`.
+    :c:func:`PyUnicodeWriter_WriteASCII(writer, str) <PyUnicodeWriter_WriteASCII>`.
   * :c:func:`!_PyUnicodeWriter_WriteLatin1String`:
     replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with
     :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) <PyUnicodeWriter_WriteUTF8>`.

Lib/reprlib.py

@@ -181,7 +181,22 @@ def repr_str(self, x, level):
         return s
 
     def repr_int(self, x, level):
-        s = builtins.repr(x) # XXX Hope this isn't too slow...
+        try:
+            s = builtins.repr(x)
+        except ValueError as exc:
+            assert 'sys.set_int_max_str_digits()' in str(exc)
+            # Those imports must be deferred due to Python's build system
+            # where the reprlib module is imported before the math module.
+            import math, sys
+            # Integers with more than sys.get_int_max_str_digits() digits
+            # are rendered differently as their repr() raises a ValueError.
+            # See https://github.com/python/cpython/issues/135487.
+            k = 1 + int(math.log10(abs(x)))
+            # Note: math.log10(abs(x)) may be overestimated or underestimated,
+            # but for simplicity, we do not compute the exact number of digits.
+            max_digits = sys.get_int_max_str_digits()
+            return (f'<{x.__class__.__name__} instance with roughly {k} '
+                    f'digits (limit at {max_digits}) at 0x{id(x):x}>')
         if len(s) > self.maxlong:
             i = max(0, (self.maxlong-3)//2)
             j = max(0, self.maxlong-3-i)

Lib/test/libregrtest/main.py

@@ -190,6 +190,12 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList
 
         strip_py_suffix(tests)
 
+        exclude_tests = set()
+        if self.exclude:
+            for arg in self.cmdline_args:
+                exclude_tests.add(arg)
+            self.cmdline_args = []
+
         if self.pgo:
             # add default PGO tests if no tests are specified
             setup_pgo_tests(self.cmdline_args, self.pgo_extended)
@@ -200,17 +206,15 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList
         if self.tsan_parallel:
             setup_tsan_parallel_tests(self.cmdline_args)
 
-        exclude_tests = set()
-        if self.exclude:
-            for arg in self.cmdline_args:
-                exclude_tests.add(arg)
-            self.cmdline_args = []
-
         alltests = findtests(testdir=self.test_dir,
                              exclude=exclude_tests)
 
         if not self.fromfile:
             selected = tests or self.cmdline_args
+            if exclude_tests:
+                # Support "--pgo/--tsan -x test_xxx" command
+                selected = [name for name in selected
+                            if name not in exclude_tests]
             if selected:
                 selected = split_test_packages(selected)
             else:

Lib/test/test_regrtest.py

@@ -2346,6 +2346,17 @@ def check(output):
         output = self.run_tests('-j1', '-v', testname, env=env, isolated=False)
         check(output)
 
+    def test_pgo_exclude(self):
+        # Get PGO tests
+        output = self.run_tests('--pgo', '--list-tests')
+        pgo_tests = output.strip().split()
+
+        # Exclude test_re
+        output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re')
+        tests = output.strip().split()
+        self.assertNotIn('test_re', tests)
+        self.assertEqual(len(tests), len(pgo_tests) - 1)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):

Lib/test/test_reprlib.py

@@ -151,14 +151,38 @@ def test_frozenset(self):
         eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})")
 
     def test_numbers(self):
-        eq = self.assertEqual
-        eq(r(123), repr(123))
-        eq(r(123), repr(123))
-        eq(r(1.0/3), repr(1.0/3))
-
-        n = 10**100
-        expected = repr(n)[:18] + "..." + repr(n)[-19:]
-        eq(r(n), expected)
+        for x in [123, 1.0 / 3]:
+            self.assertEqual(r(x), repr(x))
+
+        max_digits = sys.get_int_max_str_digits()
+        for k in [100, max_digits - 1]:
+            with self.subTest(f'10 ** {k}', k=k):
+                n = 10 ** k
+                expected = repr(n)[:18] + "..." + repr(n)[-19:]
+                self.assertEqual(r(n), expected)
+
+        def re_msg(n, d):
+            return (rf'<{n.__class__.__name__} instance with roughly {d} '
+                    rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>')
+
+        k = max_digits
+        with self.subTest(f'10 ** {k}', k=k):
+            n = 10 ** k
+            self.assertRaises(ValueError, repr, n)
+            self.assertRegex(r(n), re_msg(n, k + 1))
+
+        for k in [max_digits + 1, 2 * max_digits]:
+            self.assertGreater(k, 100)
+            with self.subTest(f'10 ** {k}', k=k):
+                n = 10 ** k
+                self.assertRaises(ValueError, repr, n)
+                self.assertRegex(r(n), re_msg(n, k + 1))
+            with self.subTest(f'10 ** {k} - 1', k=k):
+                n = 10 ** k - 1
+                # Here, since math.log10(n) == math.log10(n-1),
+                # the number of digits of n - 1 is overestimated.
+                self.assertRaises(ValueError, repr, n)
+                self.assertRegex(r(n), re_msg(n, k + 1))
 
     def test_instance(self):
         eq = self.assertEqual

Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst

@@ -0,0 +1,2 @@
+Restrict the trashcan mechanism to GC'ed objects and untrack them while in
+the trashcan to prevent the GC and trashcan mechanisms conflicting.

Misc/NEWS.d/next/Library/2025-06-14-12-06-55.gh-issue-135487.KdVFff.rst

@@ -0,0 +1,2 @@
+Fix :meth:`!reprlib.Repr.repr_int` when given integers with more than
+:func:`sys.get_int_max_str_digits` digits. Patch by Bénédikt Tran.

Misc/NEWS.d/next/Tests/2025-06-19-15-29-38.gh-issue-135494.FVl9a0.rst

@@ -0,0 +1,2 @@
+Fix regrtest to support excluding tests from ``--pgo`` tests. Patch by
+Victor Stinner.

Objects/object.c

@@ -3034,57 +3034,28 @@ Py_ReprLeave(PyObject *obj)
 
 /* Trashcan support. */
 
-#ifndef Py_GIL_DISABLED
-/* We need to store a pointer in the refcount field of
- * an object. It is important that we never store 0 (NULL).
-* It is also important to not make the object appear immortal,
-* or it might be untracked by the cycle GC. */
-static uintptr_t
-pointer_to_safe_refcount(void *ptr)
-{
-    uintptr_t full = (uintptr_t)ptr;
-    assert((full & 3) == 0);
-#if SIZEOF_VOID_P > 4
-    uint32_t refcnt = (uint32_t)full;
-    if (refcnt >= (uint32_t)_Py_IMMORTAL_MINIMUM_REFCNT) {
-        full = full - ((uintptr_t)_Py_IMMORTAL_MINIMUM_REFCNT) + 1;
-    }
-    return full + 2;
-#else
-    // Make the top two bits 0, so it appears mortal.
-    return (full >> 2) + 1;
-#endif
-}
-
-static void *
-safe_refcount_to_pointer(uintptr_t refcnt)
-{
-#if SIZEOF_VOID_P > 4
-    if (refcnt & 1) {
-        refcnt += _Py_IMMORTAL_MINIMUM_REFCNT - 1;
-    }
-    return (void *)(refcnt - 2);
-#else
-    return (void *)((refcnt -1) << 2);
-#endif
-}
-#endif
-
 /* Add op to the gcstate->trash_delete_later list.  Called when the current
- * call-stack depth gets large.  op must be a currently untracked gc'ed
- * object, with refcount 0.  Py_DECREF must already have been called on it.
+ * call-stack depth gets large.  op must be a gc'ed object, with refcount 0.
+ *  Py_DECREF must already have been called on it.
  */
 void
 _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op)
 {
     _PyObject_ASSERT(op, Py_REFCNT(op) == 0);
+    PyTypeObject *tp = Py_TYPE(op);
+    assert(tp->tp_flags & Py_TPFLAGS_HAVE_GC);
+    int tracked = 0;
+    if (tp->tp_is_gc == NULL || tp->tp_is_gc(op)) {
+        tracked = _PyObject_GC_IS_TRACKED(op);
+        if (tracked) {
+            _PyObject_GC_UNTRACK(op);
+        }
+    }
+    uintptr_t tagged_ptr = ((uintptr_t)tstate->delete_later) | tracked;
 #ifdef Py_GIL_DISABLED
-    op->ob_tid = (uintptr_t)tstate->delete_later;
+    op->ob_tid = tagged_ptr;
 #else
-    /* Store the delete_later pointer in the refcnt field. */
-    uintptr_t refcnt = pointer_to_safe_refcount(tstate->delete_later);
-    *((uintptr_t*)op) = refcnt;
-    assert(!_Py_IsImmortal(op));
+    _Py_AS_GC(op)->_gc_next = tagged_ptr;
 #endif
     tstate->delete_later = op;
 }
@@ -3099,17 +3070,17 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate)
         destructor dealloc = Py_TYPE(op)->tp_dealloc;
 
 #ifdef Py_GIL_DISABLED
-        tstate->delete_later = (PyObject*) op->ob_tid;
+        uintptr_t tagged_ptr = op->ob_tid;
         op->ob_tid = 0;
         _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED);
 #else
-        /* Get the delete_later pointer from the refcnt field.
-         * See _PyTrash_thread_deposit_object(). */
-        uintptr_t refcnt = *((uintptr_t*)op);
-        tstate->delete_later = safe_refcount_to_pointer(refcnt);
-        op->ob_refcnt = 0;
+        uintptr_t tagged_ptr = _Py_AS_GC(op)->_gc_next;
+        _Py_AS_GC(op)->_gc_next = 0;
 #endif
-
+        tstate->delete_later = (PyObject *)(tagged_ptr & ~1);
+        if (tagged_ptr & 1) {
+            _PyObject_GC_TRACK(op);
+        }
         /* Call the deallocator directly.  This used to try to
          * fool Py_DECREF into calling it indirectly, but
          * Py_DECREF was already called on this object, and in
@@ -3183,10 +3154,11 @@ void
 _Py_Dealloc(PyObject *op)
 {
     PyTypeObject *type = Py_TYPE(op);
+    unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC;
     destructor dealloc = type->tp_dealloc;
     PyThreadState *tstate = _PyThreadState_GET();
     intptr_t margin = _Py_RecursionLimit_GetMargin(tstate);
-    if (margin < 2) {
+    if (margin < 2 && gc_flag) {
         _PyTrash_thread_deposit_object(tstate, (PyObject *)op);
         return;
     }
@@ -3232,7 +3204,7 @@ _Py_Dealloc(PyObject *op)
     Py_XDECREF(old_exc);
     Py_DECREF(type);
 #endif
-    if (tstate->delete_later && margin >= 4) {
+    if (tstate->delete_later && margin >= 4 && gc_flag) {
         _PyTrash_thread_destroy_chain(tstate);
     }
 }