diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index d35e905b2e72df..e3ad4f4cdc52cc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -2287,7 +2287,7 @@ The C-API provides a basic mutual exclusion lock. should not be used to make concurrency control decisions, as the lock state may change immediately after the check. - .. versionadded:: next + .. versionadded:: 3.14 .. _python-critical-section-api: @@ -2372,7 +2372,7 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. On the default build, this macro expands to ``{``. - .. versionadded:: next + .. versionadded:: 3.14 .. c:macro:: Py_END_CRITICAL_SECTION() @@ -2418,7 +2418,7 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. On the default build, this macro expands to ``{``. - .. versionadded:: next + .. versionadded:: 3.14 .. c:macro:: Py_END_CRITICAL_SECTION2() diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 77b5e3c0876bbb..76a1e9f528dc70 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -5,11 +5,12 @@ Support for Perf Maps ---------------------- -On supported platforms (as of this writing, only Linux), the runtime can take +On supported platforms (Linux and macOS), the runtime can take advantage of *perf map files* to make Python functions visible to an external -profiling tool (such as `perf `_). -A running process may create a file in the ``/tmp`` directory, which contains entries -that can map a section of executable code to a name. This interface is described in the +profiling tool (such as `perf `_ or +`samply `_). A running process may create a +file in the ``/tmp`` directory, which contains entries that can map a section +of executable code to a name. This interface is described in the `documentation of the Linux Perf tool `_. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 96d757ac452b5e..fc4772bbccab57 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -2,34 +2,35 @@ .. _perf_profiling: -============================================== -Python support for the Linux ``perf`` profiler -============================================== +======================================================== +Python support for the ``perf map`` compatible profilers +======================================================== :author: Pablo Galindo -`The Linux perf profiler `_ -is a very powerful tool that allows you to profile and obtain -information about the performance of your application. -``perf`` also has a very vibrant ecosystem of tools -that aid with the analysis of the data that it produces. +`The Linux perf profiler `_ and +`samply `_ are powerful tools that allow you to +profile and obtain information about the performance of your application. +Both tools have vibrant ecosystems that aid with the analysis of the data they produce. -The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only gets information about native symbols, that is, the names of +The main problem with using these profilers with Python applications is that +they only get information about native symbols, that is, the names of functions and procedures written in C. This means that the names and file names -of Python functions in your code will not appear in the output of ``perf``. +of Python functions in your code will not appear in the profiler output. Since Python 3.12, the interpreter can run in a special mode that allows Python -functions to appear in the output of the ``perf`` profiler. When this mode is +functions to appear in the output of compatible profilers. When this mode is enabled, the interpreter will interpose a small piece of code compiled on the -fly before the execution of every Python function and it will teach ``perf`` the +fly before the execution of every Python function and it will teach the profiler the relationship between this piece of code and the associated Python function using :doc:`perf map files <../c-api/perfmaps>`. .. note:: - Support for the ``perf`` profiler is currently only available for Linux on - select architectures. Check the output of the ``configure`` build step or + Support for profiling is available on Linux and macOS on select architectures. + Perf is available on Linux, while samply can be used on both Linux and macOS. + samply support on macOS is available starting from Python 3.15. + Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -148,6 +149,31 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: +Using the samply profiler +------------------------- + +samply is a modern profiler that can be used as an alternative to perf. +It uses the same perf map files that Python generates, making it compatible +with Python's profiling support. samply is particularly useful on macOS +where perf is not available. + +To use samply with Python, first install it following the instructions at +https://github.com/mstange/samply, then run:: + + $ samply record PYTHONPERFSUPPORT=1 python my_script.py + +This will open a web interface where you can analyze the profiling data +interactively. The advantage of samply is that it provides a modern +web-based interface for analyzing profiling data and works on both Linux +and macOS. + +On macOS, samply support requires Python 3.15 or later. Also on macOS, samply +can't profile signed Python executables due to restrictions by macOS. You can +profile with Python binaries that you've compiled yourself, or which are +unsigned or locally-signed (such as anything installed by Homebrew). In +order to attach to running processes on macOS, run ``samply setup`` once (and +every time samply is updated) to self-sign the samply binary. + How to enable ``perf`` profiling support ---------------------------------------- diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 7cc50bccb3724a..c66308e61e89da 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -385,50 +385,77 @@ The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 - - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 + +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - ``PYTHON_MANAGER_DEFAULT`` + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` + - True to allow automatic installs when specifying a particular runtime + to launch. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -492,7 +519,7 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no @@ -638,46 +665,64 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 - - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 + +.. rubric:: Administrative configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Description + + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. .. _install-freethreaded-windows: @@ -716,70 +761,89 @@ issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 - - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " - "``pip`` gives me a ""command not found"" error when I type it in my - terminal.","Have you activated a virtual environment? Run the - ``.venv\Scripts\activate`` script in your terminal to activate. - " - "","The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, or alternatively - the ``python -m pip install --force pip`` command will recreate the - executables and show you the path to add to :envvar:`PATH`. These scripts are - separated for each runtime, and so you may need to add multiple paths. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 + +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your ``PYTHON_MANAGER_DEFAULT`` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, + or alternatively the ``python -m pip install --force pip`` command + will recreate the executables and show you the path to + add to :envvar:`PATH`. + These scripts are separated for each runtime, and so you may need to + add multiple paths. .. _windows-embeddable: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index e45a2bfa485c50..44ee2bbeb7761f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3035,6 +3035,7 @@ Porting to Python 3.14 * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt` * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer` * ``_Py_fopen_obj()``: :c:func:`Py_fopen` + * ``PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` The `pythoncapi-compat project`_ can be used to get most of these new functions on Python 3.13 and older. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 4d4fb77ad4f030..e8e2c1ed6047bf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -504,8 +504,6 @@ Porting to Python 3.15 * Private functions promoted to public C APIs: - * ``PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` - The |pythoncapi_compat_project| can be used to get most of these new functions on Python 3.14 and older. diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 80a262c18a5ad2..e8718decf6d6e9 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -762,6 +762,7 @@ def collect_support(info_add): 'is_emscripten', 'is_jython', 'is_wasi', + 'is_wasm32', ) copy_attributes(info_add, support, 'support.%s', attributes) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 196a2e5c600e24..100438bf71d3a6 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -569,6 +569,9 @@ def skip_android_selinux(name): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +# Use is_wasm32 as a generic check for WebAssembly platforms. +is_wasm32 = is_emscripten or is_wasi + def skip_emscripten_stack_overflow(): return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") @@ -3148,7 +3151,7 @@ def linked_to_musl(): # emscripten (at least as far as we're concerned) and wasi use musl, # but platform doesn't know how to get the version, so set it to zero. - if is_emscripten or is_wasi: + if is_wasm32: _linked_to_musl = (0, 0, 0) return _linked_to_musl diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 691fd0bb98c097..bc04963f5ad613 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -5,7 +5,7 @@ import string import unittest -from test.support import is_android, is_apple_mobile, is_emscripten, is_wasi +from test.support import is_android, is_apple_mobile, is_wasm32 class FormatTestsBase: @@ -91,7 +91,7 @@ def test_implementation(self): @unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') -@unittest.skipIf(is_wasi or is_emscripten, 'Feature not available on WebAssembly builds') +@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): """Test CPython's install details file implementation.""" diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6e34094c5aa422..abbd5f1ed5f12f 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -35,7 +35,7 @@ cpython_only, is_apple_mobile, is_emscripten, - is_wasi, + is_wasm32, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, @@ -1257,7 +1257,7 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 16d30e3ca2d17d..a1105aae6351b6 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -17,7 +17,7 @@ from test.support import import_helper from test.support import cpython_only -from test.support import is_emscripten, is_wasi +from test.support import is_emscripten, is_wasi, is_wasm32 from test.support import infinite_recursion from test.support import os_helper from test.support.os_helper import TESTFN, FS_NONASCII, FakePath @@ -3158,7 +3158,7 @@ def test_absolute_posix(self): self.assertEqual(str(P('//a/b').absolute()), '//a/b') @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix @@ -3189,7 +3189,7 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index d4c6fe0124af18..647c32656abd6d 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,5 +1,5 @@ import os -import sys +import sysconfig import unittest try: @@ -7,10 +7,14 @@ except ImportError: raise unittest.SkipTest("requires _testinternalcapi") +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 -if sys.platform != 'linux': - raise unittest.SkipTest('Linux only') - +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 4836f38c388c05..ed4fe8a140879d 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,6 +1,6 @@ import unittest from test.support import ( - is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_wasm32, reap_children, verbose ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -8,7 +8,7 @@ # Skip these tests if termios is not available import_module('termios') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("pty is not available on this platform") import errno diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 005526d994bd62..3b50ead00bdd31 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -33,7 +33,7 @@ assert_python_failure, spawn_python) from test.support import threading_helper from test.support import (reap_children, captured_stdout, - captured_stderr, is_emscripten, is_wasi, + captured_stderr, is_wasm32, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test.test_pydoc import pydoc_mod @@ -2081,7 +2081,7 @@ def test_html_doc_routines_in_module(self): @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py new file mode 100644 index 00000000000000..ec0ed37ffd047b --- /dev/null +++ b/Lib/test/test_samply_profiler.py @@ -0,0 +1,244 @@ +import unittest +import subprocess +import sys +import sysconfig +import os +import pathlib +from test import support +from test.support.script_helper import ( + make_script, +) +from test.support.os_helper import temp_dir + + +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + +if support.check_sanitizer(address=True, memory=True, ub=True, function=True): + # gh-109580: Skip the test because it does crash randomly if Python is + # built with ASAN. + raise unittest.SkipTest("test crash randomly on ASAN/MSAN/UBSAN build") + + +def supports_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 + + +if not supports_trampoline_profiling(): + raise unittest.SkipTest("perf trampoline profiling not supported") + + +def samply_command_works(): + try: + cmd = ["samply", "--help"] + except (subprocess.SubprocessError, OSError): + return False + + # Check that we can run a simple samply run + with temp_dir() as script_dir: + try: + output_file = script_dir + "/profile.json.gz" + cmd = ( + "samply", + "record", + "--save-only", + "--output", + output_file, + sys.executable, + "-c", + 'print("hello")', + ) + env = {**os.environ, "PYTHON_JIT": "0"} + stdout = subprocess.check_output( + cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + ) + except (subprocess.SubprocessError, OSError): + return False + + if "hello" not in stdout: + return False + + return True + + +def run_samply(cwd, *args, **env_vars): + env = os.environ.copy() + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + output_file = cwd + "/profile.json.gz" + base_cmd = ( + "samply", + "record", + "--save-only", + "-o", output_file, + ) + proc = subprocess.run( + base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + if proc.returncode: + print(proc.stderr, file=sys.stderr) + raise ValueError(f"Samply failed with return code {proc.returncode}") + + import gzip + with gzip.open(output_file, mode="rt", encoding="utf-8") as f: + return f.read() + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerMixin: + def run_samply(self, script_dir, perf_mode, script): + raise NotImplementedError() + + def test_python_calls_appear_in_the_stack_if_perf_activated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply(script_dir, script) + + self.assertIn(f"py::foo:{script}", output) + self.assertIn(f"py::bar:{script}", output) + self.assertIn(f"py::baz:{script}", output) + + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): + with temp_dir() as script_dir: + code = """if 1: + def foo(n): + x = 0 + for i in range(n): + x += i + + def bar(n): + foo(n) + + def baz(n): + bar(n) + + baz(10000000) + """ + script = make_script(script_dir, "perftest", code) + output = self.run_samply( + script_dir, script, activate_trampoline=False + ) + + self.assertNotIn(f"py::foo:{script}", output) + self.assertNotIn(f"py::bar:{script}", output) + self.assertNotIn(f"py::baz:{script}", output) + + +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfiler(unittest.TestCase, TestSamplyProfilerMixin): + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + + def test_pre_fork_compile(self): + code = """if 1: + import sys + import os + import sysconfig + from _testinternalcapi import ( + compile_perf_trampoline_entry, + perf_trampoline_set_persist_after_fork, + ) + + def foo_fork(): + pass + + def bar_fork(): + foo_fork() + + def foo(): + import time; time.sleep(1) + + def bar(): + foo() + + def compile_trampolines_for_all_functions(): + perf_trampoline_set_persist_after_fork(1) + for _, obj in globals().items(): + if callable(obj) and hasattr(obj, '__code__'): + compile_perf_trampoline_entry(obj.__code__) + + if __name__ == "__main__": + compile_trampolines_for_all_functions() + pid = os.fork() + if pid == 0: + print(os.getpid()) + bar_fork() + else: + bar() + """ + + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, "-Xperf", script], + universal_newlines=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0) + self.assertNotIn("Error:", stderr) + child_pid = int(stdout.strip()) + perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") + perf_child_file = pathlib.Path(f"/tmp/perf-{child_pid}.map") + self.assertTrue(perf_file.exists()) + self.assertTrue(perf_child_file.exists()) + + perf_file_contents = perf_file.read_text() + self.assertIn(f"py::foo:{script}", perf_file_contents) + self.assertIn(f"py::bar:{script}", perf_file_contents) + self.assertIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", perf_file_contents) + + child_perf_file_contents = perf_child_file.read_text() + self.assertIn(f"py::foo_fork:{script}", child_perf_file_contents) + self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) + + # Pre-compiled perf-map entries of a forked process must be + # identical in both the parent and child perf-map files. + perf_file_lines = perf_file_contents.split("\n") + for line in perf_file_lines: + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: + self.assertIn(line, child_perf_file_contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 92b3ef26cd979a..9ec382afb65fe4 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -787,7 +787,7 @@ def test_get_signal_name(self): def test_linked_to_musl(self): linked = support.linked_to_musl() self.assertIsNotNone(linked) - if support.is_wasi or support.is_emscripten: + if support.is_wasm32: self.assertTrue(linked) # The value is cached, so make sure it returns the same value again. self.assertIs(linked, support.linked_to_musl()) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d62f3fba2d1a94..3c18c9c2900ad7 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -21,7 +21,7 @@ from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, - is_emscripten, is_wasi, + is_wasm32, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, @@ -42,7 +42,7 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() diff --git a/Misc/ACKS b/Misc/ACKS index 35826bd713c0f6..745f472474cd9d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -43,6 +43,7 @@ Ray Allen Billy G. Allie Jamiel Almeida Kevin Altis +Nazım Can Altınova Samy Lahfa Skyler Leigh Amador Joe Amenta diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst new file mode 100644 index 00000000000000..470f3311526e83 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-09-11-15-42.gh-issue-136459.m4Udh8.rst @@ -0,0 +1,3 @@ +Add support for perf trampoline on macOS, to allow profilers wit JIT map +support to read Python calls. While profiling, ``PYTHONPERFSUPPORT=1`` can +be appended to enable the trampoline. diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index a14e68c0e81932..93adae3d99038f 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,5 +1,9 @@ .text +#if defined(__APPLE__) + .globl __Py_trampoline_func_start +#else .globl _Py_trampoline_func_start +#endif # The following assembly is equivalent to: # PyObject * # trampoline(PyThreadState *ts, _PyInterpreterFrame *f, @@ -7,7 +11,11 @@ # { # return evaluator(ts, f, throwflag); # } +#if defined(__APPLE__) +__Py_trampoline_func_start: +#else _Py_trampoline_func_start: +#endif #ifdef __x86_64__ #if defined(__CET__) && (__CET__ & 1) endbr64 @@ -35,9 +43,14 @@ _Py_trampoline_func_start: addi sp,sp,16 jr ra #endif +#if defined(__APPLE__) + .globl __Py_trampoline_func_end +__Py_trampoline_func_end: +#else .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +#endif # Note for indicating the assembly code supports CET #if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) .section .note.gnu.property,"a" diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 469882d9b2f025..3dcbcd34003102 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -66,7 +66,9 @@ #ifdef PY_HAVE_PERF_TRAMPOLINE /* Standard library includes for perf jitdump implementation */ -#include // ELF architecture constants +#if defined(__linux__) +# include // ELF architecture constants +#endif #include // File control operations #include // Standard I/O operations #include // Standard library functions @@ -74,7 +76,9 @@ #include // System data types #include // System calls (sysconf, getpid) #include // Time functions (gettimeofday) -#include // System call interface +#if defined(__linux__) +# include // System call interface +#endif // ============================================================================= // CONSTANTS AND CONFIGURATION @@ -101,6 +105,22 @@ * based on the actual unwind information requirements. */ + +/* These constants are defined inside , which we can't use outside of linux. */ +#if !defined(__linux__) +# if defined(__i386__) || defined(_M_IX86) +# define EM_386 3 +# elif defined(__arm__) || defined(_M_ARM) +# define EM_ARM 40 +# elif defined(__x86_64__) || defined(_M_X64) +# define EM_X86_64 62 +# elif defined(__aarch64__) +# define EM_AARCH64 183 +# elif defined(__riscv) +# define EM_RISCV 243 +# endif +#endif + /* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api @@ -194,7 +214,7 @@ struct BaseEvent { typedef struct { struct BaseEvent base; // Common event header uint32_t process_id; // Process ID where code was generated - uint32_t thread_id; // Thread ID where code was generated + uint64_t thread_id; // Thread ID where code was generated uint64_t vma; // Virtual memory address where code is loaded uint64_t code_address; // Address of the actual machine code uint64_t code_size; // Size of the machine code in bytes @@ -1035,6 +1055,10 @@ static void* perf_map_jit_init(void) { return NULL; // Failed to get page size } +#if defined(__APPLE__) + // On macOS, samply uses a preload to find jitdumps and this mmap can be slow. + perf_jit_map_state.mapped_buffer = NULL; +#else /* * Map the first page of the jitdump file * @@ -1057,6 +1081,7 @@ static void* perf_map_jit_init(void) { close(fd); return NULL; // Memory mapping failed } +#endif perf_jit_map_state.mapped_size = page_size; @@ -1263,7 +1288,11 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); +#if defined(__APPLE__) + pthread_threadid_np(NULL, &ev.thread_id); +#else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call +#endif ev.vma = base; // Virtual memory address ev.code_address = base; // Same as VMA for our use case ev.code_size = size; diff --git a/configure b/configure index 8db2e9c46abba2..74df430d10d6bd 100755 --- a/configure +++ b/configure @@ -13816,6 +13816,8 @@ case $PLATFORM_TRIPLET in #( perf_trampoline=yes ;; #( aarch64-linux-gnu) : perf_trampoline=yes ;; #( + darwin) : + perf_trampoline=yes ;; #( *) : perf_trampoline=no ;; diff --git a/configure.ac b/configure.ac index c839dd65a5fc5a..29e36f217402f1 100644 --- a/configure.ac +++ b/configure.ac @@ -3692,12 +3692,13 @@ case "$ac_sys_system" in esac AC_MSG_RESULT([$SHLIBS]) -dnl perf trampoline is Linux specific and requires an arch-specific +dnl perf trampoline is Linux and macOS specific and requires an arch-specific dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) AS_CASE([$PLATFORM_TRIPLET], [x86_64-linux-gnu], [perf_trampoline=yes], [aarch64-linux-gnu], [perf_trampoline=yes], + [darwin], [perf_trampoline=yes], [perf_trampoline=no] ) AC_MSG_RESULT([$perf_trampoline])