diff --git a/docs/changelog/3391.feature.rst b/docs/changelog/3391.feature.rst new file mode 100644 index 000000000..e9172b8b0 --- /dev/null +++ b/docs/changelog/3391.feature.rst @@ -0,0 +1,3 @@ +Add support for free-threaded python builds. +Factors like ``py313t`` will only pick builds with the GIL disabled while factors without trailing ``t`` will only pick +builds without no-GIL support. diff --git a/docs/config.rst b/docs/config.rst index 8ea971c56..122627ee4 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -542,11 +542,12 @@ Base options - ✅ - ✅ - ❌ + * - PYTHON_GIL + - ✅ + - ✅ + - ✅ - - - More environment variable-related information - can be found in :ref:`environment variable substitutions`. + More environment variable-related information can be found in :ref:`environment variable substitutions`. .. conf:: :keys: set_env, setenv @@ -834,6 +835,7 @@ Python options Python version for a tox environment. If not specified, the virtual environments factors (e.g. name part) will be used to automatically set one. For example, ``py310`` means ``python3.10``, ``py3`` means ``python3`` and ``py`` means ``python``. If the name does not match this pattern the same Python version tox is installed into will be used. + A base interpreter ending with ``t`` means that only free threaded Python implementations are accepted. .. versionchanged:: 3.1 @@ -866,6 +868,29 @@ Python options The Python executable from within the tox environment. +.. conf:: + :keys: py_dot_ver + :constant: + :version_added: 4.0.10 + + Major.Minor version of the Python interpreter in the tox environment (e.g., ``3.13``). + +.. conf:: + :keys: py_impl + :constant: + :version_added: 4.0.10 + + Name of the Python implementation in the tox environment in lowercase (e.g., ``cpython``, ``pypy``). + +.. conf:: + :keys: py_free_threaded + :constant: + :version_added: 4.26 + + ``True`` if the Python interpreter in the tox environment is an experimental free-threaded CPython build, + else ``False``. + + Python run ~~~~~~~~~~ .. conf:: diff --git a/src/tox/session/env_select.py b/src/tox/session/env_select.py index 1949aa173..6bbe58dc2 100644 --- a/src/tox/session/env_select.py +++ b/src/tox/session/env_select.py @@ -136,7 +136,7 @@ class _ToxEnvInfo: package_skip: tuple[str, Skip] | None = None #: if set the creation of the packaging environment failed -_DYNAMIC_ENV_FACTORS = re.compile(r"(pypy|py|cython|)((\d(\.\d+(\.\d+)?)?)|\d+)?") +_DYNAMIC_ENV_FACTORS = re.compile(r"(pypy|py|cython|)(((\d(\.\d+(\.\d+)?)?)|\d+)t?)?") _PY_PRE_RELEASE_FACTOR = re.compile(r"alpha|beta|rc\.\d+") diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index c90021252..8c0b19510 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -222,6 +222,7 @@ def _default_pass_env(self) -> list[str]: # noqa: PLR6301 "FORCE_COLOR", # force color output "NO_COLOR", # disable color output "NETRC", # used by pip and netrc modules + "PYTHON_GIL", # allows controlling python gil ] if sys.stdout.isatty(): # if we're on a interactive shell pass on the TERM env.append("TERM") diff --git a/src/tox/tox_env/python/api.py b/src/tox/tox_env/python/api.py index 37bfba55f..db59259fb 100644 --- a/src/tox/tox_env/python/api.py +++ b/src/tox/tox_env/python/api.py @@ -5,9 +5,11 @@ import logging import re import sys +import sysconfig from abc import ABC, abstractmethod +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, List, NamedTuple, cast +from typing import TYPE_CHECKING, Any, List, NamedTuple from virtualenv.discovery.py_spec import PythonSpec @@ -26,13 +28,15 @@ class VersionInfo(NamedTuple): serial: int -class PythonInfo(NamedTuple): +@dataclass(frozen=True) +class PythonInfo: implementation: str version_info: VersionInfo version: str is_64: bool platform: str extra: dict[str, Any] + free_threaded: bool = False @property def version_no_dot(self) -> str: @@ -51,11 +55,14 @@ def version_dot(self) -> str: r""" ^(?!py$) # don't match 'py' as it doesn't provide any info (?Ppy|pypy|cpython|jython|graalpy|rustpython|ironpython) # the interpreter; most users will simply use 'py' - (?P[2-9]\.?[0-9]?[0-9]?)?$ # the version; one of: MAJORMINOR, MAJOR.MINOR + (?: + (?P[2-9]\.?[0-9]?[0-9]?) # the version; one of: MAJORMINOR, MAJOR.MINOR + (?Pt?) # version followed by t for free-threading + )?$ """, re.VERBOSE, ) -PY_FACTORS_RE_EXPLICIT_VERSION = re.compile(r"^((?Pcpython|pypy)-)?(?P[2-9]\.[0-9]+)$") +PY_FACTORS_RE_EXPLICIT_VERSION = re.compile(r"^((?Pcpython|pypy)-)?(?P[2-9]\.[0-9]+)(?Pt?)$") class Python(ToxEnv, ABC): @@ -100,6 +107,7 @@ def validate_base_python(value: list[str]) -> list[str]: ) self.conf.add_constant("py_dot_ver", ".", value=self.py_dot_ver) self.conf.add_constant("py_impl", "python implementation", value=self.py_impl) + self.conf.add_constant("py_free_threaded", "is no-gil interpreted", value=self.py_free_threaded) def _default_set_env(self) -> dict[str, str]: env = super()._default_set_env() @@ -111,6 +119,9 @@ def _default_set_env(self) -> dict[str, str]: def py_dot_ver(self) -> str: return self.base_python.version_dot + def py_free_threaded(self) -> bool: + return self.base_python.free_threaded + def py_impl(self) -> str: return self.base_python.impl_lower @@ -145,7 +156,7 @@ def extract_base_python(cls, env_name: str) -> str | None: match = PY_FACTORS_RE_EXPLICIT_VERSION.match(env_name) if match: found = match.groupdict() - candidates.append(f"{'pypy' if found['impl'] == 'pypy' else ''}{found['version']}") + candidates.append(f"{'pypy' if found['impl'] == 'pypy' else ''}{found['version']}{found['threaded']}") else: for factor in env_name.split("-"): match = PY_FACTORS_RE.match(factor) @@ -163,7 +174,8 @@ def _python_spec_for_sys_executable(cls) -> PythonSpec: implementation = sys.implementation.name version = sys.version_info bits = "64" if sys.maxsize > 2**32 else "32" - string_spec = f"{implementation}{version.major}{version.minor}-{bits}" + threaded = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") == 1 else "" + string_spec = f"{implementation}{version.major}{version.minor}{threaded}-{bits}" return PythonSpec.from_string_spec(string_spec) @classmethod @@ -186,7 +198,7 @@ def _validate_base_python( spec_base = cls.python_spec_for_path(path) if any( getattr(spec_base, key) != getattr(spec_name, key) - for key in ("implementation", "major", "minor", "micro", "architecture") + for key in ("implementation", "major", "minor", "micro", "architecture", "free_threaded") if getattr(spec_name, key) is not None ): msg = f"env name {env_name} conflicting with base python {base_python}" @@ -290,7 +302,7 @@ def base_python(self) -> PythonInfo: raise Skip(msg) raise NoInterpreter(base_pythons) - return cast("PythonInfo", self._base_python) + return self._base_python def _get_env_journal_python(self) -> dict[str, Any]: return { diff --git a/src/tox/tox_env/python/virtual_env/api.py b/src/tox/tox_env/python/virtual_env/api.py index 861cd7847..7c56c14c6 100644 --- a/src/tox/tox_env/python/virtual_env/api.py +++ b/src/tox/tox_env/python/virtual_env/api.py @@ -146,6 +146,7 @@ def _get_python(self, base_python: list[str]) -> PythonInfo | None: # noqa: ARG is_64=(interpreter.architecture == 64), # noqa: PLR2004 platform=interpreter.platform, extra={"executable": Path(interpreter.system_executable).resolve()}, + free_threaded=interpreter.free_threaded, ) def prepend_env_var_path(self) -> list[Path]: diff --git a/tests/conftest.py b/tests/conftest.py index 7a12fbef2..c5b12d93e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import os import sys +import sysconfig from pathlib import Path from typing import TYPE_CHECKING, Callable, Iterator, Protocol, Sequence from unittest.mock import patch @@ -100,6 +101,7 @@ def get_python(self: VirtualEnv, base_python: list[str]) -> PythonInfo | None: is_64=True, platform=sys.platform, extra={"executable": Path(sys.executable)}, + free_threaded=sysconfig.get_config_var("Py_GIL_DISABLED") == 1, ) mocker.patch.object(VirtualEnv, "_get_python", get_python) diff --git a/tests/session/cmd/test_show_config.py b/tests/session/cmd/test_show_config.py index 5a2e9b637..6b757e7ee 100644 --- a/tests/session/cmd/test_show_config.py +++ b/tests/session/cmd/test_show_config.py @@ -135,7 +135,7 @@ def test_pass_env_config_default(tox_project: ToxProjectCreator, stdout_is_atty: + (["PROGRAMDATA"] if is_win else []) + (["PROGRAMFILES"] if is_win else []) + (["PROGRAMFILES(x86)"] if is_win else []) - + ["REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"] + + ["PYTHON_GIL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"] + (["SYSTEMDRIVE", "SYSTEMROOT", "TEMP"] if is_win else []) + (["TERM"] if stdout_is_atty else []) + (["TMP", "USERPROFILE"] if is_win else ["TMPDIR"]) diff --git a/tests/session/test_env_select.py b/tests/session/test_env_select.py index 80a0043fe..52d8030b7 100644 --- a/tests/session/test_env_select.py +++ b/tests/session/test_env_select.py @@ -261,10 +261,15 @@ def test_matches_combined_env(env_name: str, tox_project: ToxProjectCreator) -> "pypy312", "py3", "py3.12", + "py3.12t", "py312", + "py312t", "3", + "3t", "3.12", + "3.12t", "3.12.0", + "3.12.0t", ], ) def test_dynamic_env_factors_match(env: str) -> None: diff --git a/tests/tox_env/python/test_python_api.py b/tests/tox_env/python/test_python_api.py index 02b6b91ca..413725ecd 100644 --- a/tests/tox_env/python/test_python_api.py +++ b/tests/tox_env/python/test_python_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import sysconfig from types import SimpleNamespace from typing import TYPE_CHECKING, Callable from unittest.mock import patch @@ -81,30 +82,56 @@ def test_diff_msg_no_diff() -> None: ("env", "base_python"), [ ("py3", "py3"), + ("py3t", "py3t"), ("py311", "py311"), + ("py311t", "py311t"), ("py3.12", "py3.12"), + ("py3.12t", "py3.12t"), ("pypy2", "pypy2"), + ("pypy2t", "pypy2t"), ("rustpython3", "rustpython3"), + ("rustpython3t", "rustpython3t"), ("graalpy", "graalpy"), + ("graalpyt", None), ("jython", "jython"), + ("jythont", None), ("cpython3.8", "cpython3.8"), + ("cpython3.8t", "cpython3.8t"), ("ironpython2.7", "ironpython2.7"), + ("ironpython2.7t", "ironpython2.7t"), ("functional-py310", "py310"), + ("functional-py310t", "py310t"), ("bar-pypy2-foo", "pypy2"), + ("bar-foo2t-py2", "py2"), + ("bar-pypy2t-foo", "pypy2t"), ("py", None), + ("pyt", None), ("django-32", None), + ("django-32t", None), ("eslint-8.3", None), + ("eslint-8.3t", None), ("py-310", None), + ("py-310t", None), ("py3000", None), + ("py3000t", None), ("4.foo", None), + ("4.foot", None), ("310", None), + ("310t", None), ("5", None), + ("5t", None), ("2000", None), + ("2000t", None), ("4000", None), + ("4000t", None), ("3.10", "3.10"), + ("3.10t", "3.10t"), ("3.9", "3.9"), + ("3.9t", "3.9t"), ("2.7", "2.7"), + ("2.7t", "2.7t"), ("pypy-3.10", "pypy3.10"), + ("pypy-3.10t", "pypy3.10t"), ], ids=lambda a: "|".join(a) if isinstance(a, list) else str(a), ) @@ -294,13 +321,24 @@ def test_usedevelop_with_nonexistent_basepython(tox_project: ToxProjectCreator) @pytest.mark.parametrize( - ("impl", "major", "minor", "arch"), + ("impl", "major", "minor", "arch", "free_threaded"), [ - ("cpython", 3, 12, 64), - ("pypy", 3, 9, 32), + ("cpython", 3, 12, 64, None), + ("cpython", 3, 13, 64, True), + ("cpython", 3, 13, 64, False), + ("pypy", 3, 9, 32, None), ], ) -def test_python_spec_for_sys_executable(impl: str, major: int, minor: int, arch: int, mocker: MockerFixture) -> None: +def test_python_spec_for_sys_executable( # noqa: PLR0913 + impl: str, major: int, minor: int, arch: int, free_threaded: bool | None, mocker: MockerFixture +) -> None: + get_config_var_ = sysconfig.get_config_var + + def get_config_var(name: str) -> object: + if name == "Py_GIL_DISABLED": + return free_threaded + return get_config_var_(name) + version_info = SimpleNamespace(major=major, minor=minor, micro=5, releaselevel="final", serial=0) implementation = SimpleNamespace( name=impl, @@ -312,8 +350,10 @@ def test_python_spec_for_sys_executable(impl: str, major: int, minor: int, arch: mocker.patch.object(sys, "version_info", version_info) mocker.patch.object(sys, "implementation", implementation) mocker.patch.object(sys, "maxsize", 2**arch // 2 - 1) + mocker.patch.object(sysconfig, "get_config_var", get_config_var) spec = Python._python_spec_for_sys_executable() # noqa: SLF001 assert spec.implementation == impl assert spec.major == major assert spec.minor == minor assert spec.architecture == arch + assert spec.free_threaded == bool(free_threaded) diff --git a/tests/tox_env/python/test_python_runner.py b/tests/tox_env/python/test_python_runner.py index 163d4e234..2e5eea3ae 100644 --- a/tests/tox_env/python/test_python_runner.py +++ b/tests/tox_env/python/test_python_runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import sysconfig from pathlib import Path from typing import TYPE_CHECKING @@ -152,11 +153,16 @@ def test_config_skip_missing_interpreters( assert result.code == (0 if expected else -1) +SYS_PY_VER = "".join(str(i) for i in sys.version_info[0:2]) + ( + "t" if sysconfig.get_config_var("Py_GIL_DISABLED") == 1 else "" +) + + @pytest.mark.parametrize( ("skip", "env", "retcode"), [ - ("true", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0), - ("false", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0), + ("true", f"py{SYS_PY_VER}", 0), + ("false", f"py{SYS_PY_VER}", 0), ("true", "py31", -1), ("false", "py31", 1), ("true", None, 0), @@ -169,8 +175,7 @@ def test_skip_missing_interpreters_specified_env( env: str | None, retcode: int, ) -> None: - py_ver = "".join(str(i) for i in sys.version_info[0:2]) - project = tox_project({"tox.ini": f"[tox]\nenvlist=py31,py{py_ver}\n[testenv]\nusedevelop=true"}) + project = tox_project({"tox.ini": f"[tox]\nenvlist=py31,py{SYS_PY_VER}\n[testenv]\nusedevelop=true"}) args = [f"--skip-missing-interpreters={skip}"] if env: args += ["-e", env] diff --git a/tox.toml b/tox.toml index 574d9a210..f998ba10d 100644 --- a/tox.toml +++ b/tox.toml @@ -1,5 +1,5 @@ requires = ["tox>=4.24.1"] -env_list = ["fix", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "cov", "type", "docs", "pkg_meta"] +env_list = ["fix", "3.14t", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "cov", "type", "docs", "pkg_meta"] skip_missing_interpreters = true [env_run_base]