[pull] main from python:main

.github/workflows/reusable-docs.yml

@@ -66,7 +66,7 @@ jobs:
       run: |
         set -Eeuo pipefail
         # Build docs with the nit-picky option; write warnings to file
-        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html
     - name: 'Check warnings'
       if: github.event_name == 'pull_request'
       run: |
@@ -75,6 +75,18 @@ jobs:
           --fail-if-regression \
           --fail-if-improved \
           --fail-if-new-news-nit
+    - name: 'Build EPUB documentation'
+      continue-on-error: true
+      run: |
+        set -Eeuo pipefail
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub
+        pip install epubcheck
+        epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt
+    - name: 'Check for fatal errors in EPUB'
+      if: github.event_name == 'pull_request'
+      continue-on-error: true  # until gh-136155 is fixed
+      run: |
+        python Doc/tools/check-epub.py
 
   # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
   doctest:

Doc/c-api/init_config.rst

@@ -975,9 +975,7 @@ PyPreConfig
       Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option
       and the :envvar:`PYTHONUTF8` environment variable.
 
-      Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``.
-
-      Default: ``-1`` in Python config and ``0`` in isolated config.
+      Default: ``1``.
 
 
 .. _c-preinit:

Doc/conf.py

@@ -448,6 +448,7 @@
 
 epub_author = 'Python Documentation Authors'
 epub_publisher = 'Python Software Foundation'
+epub_exclude_files = ('index.xhtml', 'download.xhtml')
 
 # index pages are not valid xhtml
 # https://github.com/sphinx-doc/sphinx/issues/12359

Doc/library/hmac.rst

@@ -12,6 +12,9 @@
 --------------
 
 This module implements the HMAC algorithm as described by :rfc:`2104`.
+The interface allows to use any hash function with a *fixed* digest size.
+In particular, extendable output functions such as SHAKE-128 or SHAKE-256
+cannot be used with HMAC.
 
 
 .. function:: new(key, msg=None, digestmod)

Doc/library/os.path.rst

@@ -508,9 +508,6 @@ the :mod:`glob` module.)
    .. versionchanged:: 3.4
       Added Windows support.
 
-   .. versionchanged:: 3.6
-      Accepts a :term:`path-like object`.
-
 
 .. function:: split(path)
 

Doc/library/os.rst

@@ -108,6 +108,12 @@ Python UTF-8 Mode
 .. versionadded:: 3.7
    See :pep:`540` for more details.
 
+.. versionchanged:: next
+
+   Python UTF-8 mode is now enabled by default (:pep:`686`).
+   It may be disabled with by setting :envvar:`PYTHONUTF8=0 <PYTHONUTF8>` as
+   an environment variable or by using the :option:`-X utf8=0 <-X>` command line option.
+
 The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage
 of the UTF-8 encoding:
 
@@ -139,31 +145,22 @@ level APIs also exhibit different default behaviours:
   default so that attempting to open a binary file in text mode is likely
   to raise an exception rather than producing nonsense data.
 
-The :ref:`Python UTF-8 Mode <utf8-mode>` is enabled if the LC_CTYPE locale is
-``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read`
-function).
-
-It can be enabled or disabled using the :option:`-X utf8 <-X>` command line
-option and the :envvar:`PYTHONUTF8` environment variable.
-
-If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the
-interpreter defaults to using the current locale settings, *unless* the current
-locale is identified as a legacy ASCII-based locale (as described for
-:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or
-fails. In such legacy locales, the interpreter will default to enabling UTF-8
-mode unless explicitly instructed not to do so.
-
-The Python UTF-8 Mode can only be enabled at the Python startup. Its value
+The :ref:`Python UTF-8 Mode <utf8-mode>` is enabled by default.
+It can be disabled using the :option:`-X utf8=0 <-X>` command line
+option or the :envvar:`PYTHONUTF8=0 <PYTHONUTF8>` environment variable.
+The Python UTF-8 Mode can only be disabled at Python startup. Its value
 can be read from :data:`sys.flags.utf8_mode <sys.flags>`.
 
+If the UTF-8 mode is disabled, the interpreter defaults to using
+the current locale settings, *unless* the current locale is identified
+as a legacy ASCII-based locale (as described for :envvar:`PYTHONCOERCECLOCALE`),
+and locale coercion is either disabled or fails.
+In such legacy locales, the interpreter will default to enabling UTF-8 mode
+unless explicitly instructed not to do so.
+
 See also the :ref:`UTF-8 mode on Windows <win-utf8-mode>`
 and the :term:`filesystem encoding and error handler`.
 
-.. seealso::
-
-   :pep:`686`
-      Python 3.15 will make :ref:`utf8-mode` default.
-
 
 .. _os-procinfo:
 

Doc/tools/check-epub.py

@@ -0,0 +1,24 @@
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    wrong_directory_msg = "Must run this script from the repo root"
+    if not Path("Doc").exists() or not Path("Doc").is_dir():
+        raise RuntimeError(wrong_directory_msg)
+
+    with Path("Doc/epubcheck.txt").open(encoding="UTF-8") as f:
+        messages = [message.split(" - ") for message in f.read().splitlines()]
+
+    fatal_errors = [message for message in messages if message[0] == "FATAL"]
+
+    if fatal_errors:
+        print("\nError: must not contain fatal errors:\n")
+        for error in fatal_errors:
+            print(" - ".join(error))
+
+    return len(fatal_errors)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

Doc/using/windows.rst

@@ -1006,6 +1006,9 @@ UTF-8 mode
 ==========
 
 .. versionadded:: 3.7
+.. versionchanged:: next
+
+   Python UTF-8 mode is now enabled by default (:pep:`686`).
 
 Windows still uses legacy encodings for the system encoding (the ANSI Code
 Page).  Python uses it for the default encoding of text files (e.g.
@@ -1014,20 +1017,22 @@ Page).  Python uses it for the default encoding of text files (e.g.
 This may cause issues because UTF-8 is widely used on the internet
 and most Unix systems, including WSL (Windows Subsystem for Linux).
 
-You can use the :ref:`Python UTF-8 Mode <utf8-mode>` to change the default text
-encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode <utf8-mode>` via
-the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment
-variable.  See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and
-:ref:`setting-envvars` for how to modify environment variables.
-
-When the :ref:`Python UTF-8 Mode <utf8-mode>` is enabled, you can still use the
+The :ref:`Python UTF-8 Mode <utf8-mode>`, enabled by default, can help by
+changing the default text encoding to UTF-8.
+When the :ref:`UTF-8 mode <utf8-mode>` is enabled, you can still use the
 system encoding (the ANSI Code Page) via the "mbcs" codec.
 
-Note that adding ``PYTHONUTF8=1`` to the default environment variables
-will affect all Python 3.7+ applications on your system.
-If you have any Python 3.7+ applications which rely on the legacy
-system encoding, it is recommended to set the environment variable
-temporarily or use the ``-X utf8`` command line option.
+You can disable the :ref:`Python UTF-8 Mode <utf8-mode>` via
+the ``-X utf8=0`` command line option, or the ``PYTHONUTF8=0`` environment
+variable.  See :envvar:`PYTHONUTF8` for disabling UTF-8 mode, and
+:ref:`setting-envvars` for how to modify environment variables.
+
+.. hint::
+   Adding ``PYTHONUTF8={0,1}`` to the default environment variables
+   will affect all Python 3.7+ applications on your system.
+   If you have any Python 3.7+ applications which rely on the legacy
+   system encoding, it is recommended to set the environment variable
+   temporarily or use the ``-X utf8`` command line option.
 
 .. note::
    Even when UTF-8 mode is disabled, Python uses UTF-8 by default

Doc/whatsnew/3.15.rst

@@ -172,11 +172,35 @@ production systems where traditional profiling approaches would be too intrusive
 Other language changes
 ======================
 
+* Python now uses UTF-8_ as the default encoding, independent of the system's
+  environment. This means that I/O operations without an explicit encoding,
+  e.g. ``open('flying-circus.txt')``, will use UTF-8.
+  UTF-8 is a widely-supported Unicode_ character encoding that has become a
+  *de facto* standard for representing text, including nearly every webpage
+  on the internet, many common file formats, programming languages, and more.
+
+  This only applies when no ``encoding`` argument is given. For best
+  compatibility between versions of Python, ensure that an explicit ``encoding``
+  argument is always provided. The :ref:`opt-in encoding warning <io-encoding-warning>`
+  can be used to identify code that may be affected by this change.
+  The special special ``encoding='locale'`` argument uses the current locale
+  encoding, and has been supported since Python 3.10.
+
+  To retain the previous behaviour, Python's UTF-8 mode may be disabled with
+  the :envvar:`PYTHONUTF8=0 <PYTHONUTF8>` environment variable or the
+  :option:`-X utf8=0 <-X>` command line option.
+
+  .. seealso:: :pep:`686` for further details.
+
+  .. _UTF-8: https://en.wikipedia.org/wiki/UTF-8
+  .. _Unicode: https://home.unicode.org/
+
+  (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.)
+
 * Several error messages incorrectly using the term "argument" have been corrected.
   (Contributed by Stan Ulbrych in :gh:`133382`.)
 
 
-
 New modules
 ===========
 

Include/cpython/initconfig.h

@@ -102,15 +102,14 @@ typedef struct PyPreConfig {
 
     /* Enable UTF-8 mode? (PEP 540)
 
-       Disabled by default (equals to 0).
+      If equal to 1, use the UTF-8 encoding and use "surrogateescape" for the
+      stdin & stdout error handlers.
 
-       Set to 1 by "-X utf8" and "-X utf8=1" command line options.
-       Set to 1 by PYTHONUTF8=1 environment variable.
+      Enabled by default (equal to 1; PEP 686), or if Py_UTF8Mode=1,
+      or if "-X utf8=1" or PYTHONUTF8=1.
 
-       Set to 0 by "-X utf8=0" and PYTHONUTF8=0.
-
-       If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or
-       "POSIX", otherwise it is set to 0. Inherit Py_UTF8Mode value value. */
+       Set to 0 by "-X utf8=0" or PYTHONUTF8=0.
+    */
     int utf8_mode;
 
     /* If non-zero, enable the Python Development Mode.

Lib/_pyrepl/console.py

@@ -27,6 +27,7 @@
 import linecache
 from dataclasses import dataclass, field
 import os.path
+import re
 import sys
 
 
@@ -195,7 +196,19 @@ def runsource(self, source, filename="<input>", symbol="single"):
                 ast.PyCF_ONLY_AST,
                 incomplete_input=False,
             )
-        except (SyntaxError, OverflowError, ValueError):
+        except SyntaxError as e:
+            # If it looks like pip install was entered (a common beginner
+            # mistake), provide a hint to use the system command prompt.
+            if re.match(r"^\s*(pip3?|py(thon3?)? -m pip) install.*", source):
+                e.add_note(
+                    "The Python package manager (pip) can only be used"
+                    " outside of the Python REPL.\n"
+                    "Try the 'pip' command in a separate terminal or"
+                    " command prompt."
+                )
+            self.showsyntaxerror(filename, source=source)
+            return False
+        except (OverflowError, ValueError):
             self.showsyntaxerror(filename, source=source)
             return False
         if tree.body:

Lib/locale.py

@@ -651,7 +651,8 @@ def getpreferredencoding(do_setlocale=True):
         if sys.flags.warn_default_encoding:
             import warnings
             warnings.warn(
-                "UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.",
+                "UTF-8 Mode affects locale.getpreferredencoding(). "
+                "Consider locale.getencoding() instead.",
                 EncodingWarning, 2)
         if sys.flags.utf8_mode:
             return 'utf-8'

Lib/string/templatelib.py

@@ -1,15 +1,22 @@
 """Support for template string literals (t-strings)."""
 
-__all__ = [
-    "Interpolation",
-    "Template",
-]
-
 t = t"{0}"
 Template = type(t)
 Interpolation = type(t.interpolations[0])
 del t
 
+def convert(obj, /, conversion):
+    """Convert *obj* using formatted string literal semantics."""
+    if conversion is None:
+        return obj
+    if conversion == 'r':
+        return repr(obj)
+    if conversion == 's':
+        return str(obj)
+    if conversion == 'a':
+        return ascii(obj)
+    raise ValueError(f'invalid conversion specifier: {conversion}')
+
 def _template_unpickle(*args):
     import itertools
 

Lib/subprocess.py

@@ -380,8 +380,7 @@ def _text_encoding():
 
     if sys.flags.utf8_mode:
         return "utf-8"
-    else:
-        return locale.getencoding()
+    return locale.getencoding()
 
 
 def call(*popenargs, timeout=None, **kwargs):

Lib/test/test_cmd_line.py

@@ -300,6 +300,10 @@ def run_utf8_mode(arg):
             cmd = [sys.executable, '-X', 'utf8', '-c', code, arg]
             return subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
 
+        def run_no_utf8_mode(arg):
+            cmd = [sys.executable, '-X', 'utf8=0', '-c', code, arg]
+            return subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
+
         valid_utf8 = 'e:\xe9, euro:\u20ac, non-bmp:\U0010ffff'.encode('utf-8')
         # invalid UTF-8 byte sequences with a valid UTF-8 sequence
         # in the middle.
@@ -312,7 +316,8 @@ def run_utf8_mode(arg):
         )
         test_args = [valid_utf8, invalid_utf8]
 
-        for run_cmd in (run_default, run_c_locale, run_utf8_mode):
+        for run_cmd in (run_default, run_c_locale, run_utf8_mode,
+                        run_no_utf8_mode):
             with self.subTest(run_cmd=run_cmd):
                 for arg in test_args:
                     proc = run_cmd(arg)

Lib/test/test_embed.py

@@ -543,7 +543,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'configure_locale': True,
         'coerce_c_locale': False,
         'coerce_c_locale_warn': False,
-        'utf8_mode': False,
+        'utf8_mode': True,
     }
     if MS_WINDOWS:
         PRE_CONFIG_COMPAT.update({
@@ -560,7 +560,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         configure_locale=False,
         isolated=True,
         use_environment=False,
-        utf8_mode=False,
+        utf8_mode=True,
         dev_mode=False,
         coerce_c_locale=False,
     )
@@ -805,12 +805,6 @@ def get_expected_config(self, expected_preconfig, expected,
                         'stdio_encoding', 'stdio_errors'):
                 expected[key] = self.IGNORE_CONFIG
 
-        if not expected_preconfig['configure_locale']:
-            # UTF-8 Mode depends on the locale. There is no easy way
-            # to guess if UTF-8 Mode will be enabled or not if the locale
-            # is not configured.
-            expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG
-
         if expected_preconfig['utf8_mode'] == 1:
             if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG:
                 expected['filesystem_encoding'] = 'utf-8'

Lib/test/test_pyrepl/test_pyrepl.py

@@ -1757,3 +1757,14 @@ def test_showrefcount(self):
         output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env)
         matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output)
         self.assertEqual(len(matches), 3)
+
+    def test_detect_pip_usage_in_repl(self):
+        for pip_cmd in ("pip", "pip3", "python -m pip", "python3 -m pip"):
+            with self.subTest(pip_cmd=pip_cmd):
+                output, exit_code = self.run_repl([f"{pip_cmd} install sampleproject", "exit"])
+                self.assertIn("SyntaxError", output)
+                hint = (
+                    "The Python package manager (pip) can only be used"
+                    " outside of the Python REPL"
+                )
+                self.assertIn(hint, output)