From 4c3a891a40642854dac0b97947df47a21391b0c7 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 16 Apr 2019 07:34:49 -0700 Subject: [PATCH 01/72] Fix RunResult unicode repr errors (#1264) --- src/tox/_pytestplugin.py | 8 ++++++-- tests/unit/test_pytest_plugins.py | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 14d166bd6..7637880ff 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -176,9 +176,13 @@ def outlines(self): return err + out def __repr__(self): - return "RunResult(ret={}, args={}, out=\n{}\n, err=\n{})".format( - self.ret, " ".join(str(i) for i in self.args), self.out, self.err + res = "RunResult(ret={}, args={!r}, out=\n{}\n, err=\n{})".format( + self.ret, self.args, self.out, self.err ) + if six.PY2: + return res.encode("UTF-8") + else: + return res def output(self): return "{}\n{}\n{}".format(self.ret, self.err, self.out) diff --git a/tests/unit/test_pytest_plugins.py b/tests/unit/test_pytest_plugins.py index 873731b02..dcb6e3a06 100644 --- a/tests/unit/test_pytest_plugins.py +++ b/tests/unit/test_pytest_plugins.py @@ -4,11 +4,12 @@ """ import os +import sys import py.path import pytest -from tox._pytestplugin import _filedefs_contains, _path_parts +from tox._pytestplugin import RunResult, _filedefs_contains, _path_parts class TestInitProj: @@ -111,3 +112,15 @@ def test_on_py_path(self): ) def test_filedefs_contains(base, filedefs, target, expected): assert bool(_filedefs_contains(base, filedefs, target)) == expected + + +def test_run_result_repr(capfd): + with RunResult(["hello", "world"], capfd) as run_result: + # simulate tox writing some unicode output + stdout_buffer = getattr(sys.stdout, "buffer", sys.stdout) + stdout_buffer.write(u"\u2603".encode("UTF-8")) + + # must not `UnicodeError` on repr(...) + ret = repr(run_result) + # must be native `str`, (bytes in py2, str in py3) + assert isinstance(ret, str) From 3efa0caafd0ee21378e71584f0bbfe6dfa94eb4f Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 15 Apr 2019 11:05:23 +0100 Subject: [PATCH 02/72] move to latest images --- azure-pipelines.yml | 8 ++++++-- azure-run-tox-env.yml | 6 +++--- tox.ini | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b7b25c85f..7bde22e14 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,7 @@ variables: PYTEST_ADDOPTS: "-v -v -ra --showlocals" GIT_BRANCH: $[ coalesce(variables['System.PullRequest.SourceBranch'], variables['Build.SourceBranchName'], 'not-found') ] GIT_COMMIT_SHA: $[ coalesce(variables['System.PullRequest.SourceCommitId'], variables['Build.SourceVersion'], 'not-found') ] + PYTEST_XDIST_PROC_NR: 'auto' trigger: batch: true @@ -73,7 +74,9 @@ jobs: parameters: {tox: py27, python: 2.7, os: linux} - template: azure-run-tox-env.yml - parameters: {tox: py36, python: 3.6, os: macOs} + parameters: {tox: py37, python: 3.7, os: macOs} +- template: azure-run-tox-env.yml + parameters: {tox: py27, python: 2.7, os: macOs} - job: report_coverage pool: {vmImage: 'Ubuntu 16.04'} @@ -91,7 +94,8 @@ jobs: - linux_py27 - linux_pypy3 - linux_pypy - - macOS_py36 + - macOS_py37 + - macOS_py27 steps: - task: DownloadBuildArtifacts@0 displayName: download coverage files for run diff --git a/azure-run-tox-env.yml b/azure-run-tox-env.yml index a75feea79..f18e192d7 100644 --- a/azure-run-tox-env.yml +++ b/azure-run-tox-env.yml @@ -8,11 +8,11 @@ jobs: dependsOn: notify_build_start pool: ${{ if eq(parameters.os, 'windows') }}: - vmImage: "vs2017-win2016" + vmImage: "windows-2019" ${{ if eq(parameters.os, 'macOs') }}: - vmImage: "macOS 10.13" + vmImage: "macOS-latest" ${{ if eq(parameters.os, 'linux') }}: - vmImage: "Ubuntu 16.04" + vmImage: "Ubuntu-16.04" variables: TMPDIR: $(Build.BinariesDirectory) diff --git a/tox.ini b/tox.ini index 403466b73..c800093fa 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ skip_missing_interpreters = true description = run the tests with pytest under {basepython} setenv = PIP_DISABLE_VERSION_CHECK = 1 COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} -passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_ADDOPTS +passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_* deps = extras = testing commands = pytest \ From dc7d045d5a47af7c970ab39cfeacd4180d56813c Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 16 Apr 2019 15:36:43 +0100 Subject: [PATCH 03/72] fix flaky test --- azure-pipelines.yml | 6 +++--- azure-run-tox-env.yml | 7 +++++-- tests/unit/session/test_parallel.py | 8 ++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7bde22e14..ab544dc69 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,10 +5,10 @@ variables: CI_NAME: Azure Pipelines CI_BUILD_ID: $(Build.BuildId) CI_BUILD_URL: "https://toxdev.visualstudio.com/tox/_build/results?buildId=$(Build.BuildId)" - PYTEST_ADDOPTS: "-v -v -ra --showlocals" + PYTEST_ADDOPTS: "-v -v -ra --showlocals -k test_config" GIT_BRANCH: $[ coalesce(variables['System.PullRequest.SourceBranch'], variables['Build.SourceBranchName'], 'not-found') ] - GIT_COMMIT_SHA: $[ coalesce(variables['System.PullRequest.SourceCommitId'], variables['Build.SourceVersion'], 'not-found') ] PYTEST_XDIST_PROC_NR: 'auto' + GIT_COMMIT_SHA: $[ coalesce(variables['System.PullRequest.SourceCommitId'], variables['Build.SourceVersion'], 'not-found') ] trigger: batch: true @@ -124,7 +124,7 @@ jobs: shutil.copy(str(coverage_file), str(destination))' displayName: move coverage files into .tox - - script: "python -m pip install -U pip setuptools --user -v" + - script: "python -m pip install -U pip setuptools --user" displayName: upgrade pip - script: 'python -m pip install . -U --user' diff --git a/azure-run-tox-env.yml b/azure-run-tox-env.yml index f18e192d7..3f0c7cbba 100644 --- a/azure-run-tox-env.yml +++ b/azure-run-tox-env.yml @@ -20,6 +20,8 @@ jobs: python: ${{ parameters.python }} ${{ if notIn(parameters.python, 'pypy', 'pypy3') }}: python: "python" + ${{ if in(parameters.python, 'pypy', 'py27', 'py34') }}: + PYTHONWARNINGS: 'ignore:::pip._internal.cli.base_command' steps: # ensure the required Python versions are available @@ -32,10 +34,10 @@ jobs: - script: "$(python) -c \"import sys; print(sys.version); print(sys.executable)\"" displayName: show python information - - script: "python -m pip install -U pip setuptools --user -v" + - script: "python -m pip install -U pip setuptools --user" displayName: upgrade pip - - script: "python -m pip install -U . --user -v" + - script: "python -m pip install -U . --user" displayName: install tox - eat our own dog food - script: ${{ format('python -m tox -e {0} --notest', parameters.tox) }} @@ -63,6 +65,7 @@ jobs: condition: succeededOrFailed() - ${{ if startsWith(parameters.tox, 'py') }}: + - task: CopyFiles@2 displayName: move coverage files into staging area condition: succeededOrFailed() diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index 0033cf607..58435333c 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -184,9 +184,5 @@ def test_parallel_show_output(cmd, initproj, monkeypatch): result.assert_success() assert "stdout env" not in result.out, result.output() assert "stderr env" not in result.out, result.output() - msg = ( - "stdout always stderr always" - if sys.version_info[0] == 3 - else "stderr always stdout always" - ) - assert msg in result.out, result.output() + assert "stdout always" in result.out, result.output() + assert "stderr always" in result.out, result.output() From 88545ffa485d825aa8bfbafc3092427d7a8e4ff3 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 16 Apr 2019 16:08:19 +0100 Subject: [PATCH 04/72] remove pytest timeout having issues with pytest-xdist --- azure-pipelines.yml | 17 +++++++++-------- azure-run-tox-env.yml | 6 +++--- setup.cfg | 1 - tox.ini | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ab544dc69..6dda98580 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,10 +5,12 @@ variables: CI_NAME: Azure Pipelines CI_BUILD_ID: $(Build.BuildId) CI_BUILD_URL: "https://toxdev.visualstudio.com/tox/_build/results?buildId=$(Build.BuildId)" - PYTEST_ADDOPTS: "-v -v -ra --showlocals -k test_config" GIT_BRANCH: $[ coalesce(variables['System.PullRequest.SourceBranch'], variables['Build.SourceBranchName'], 'not-found') ] - PYTEST_XDIST_PROC_NR: 'auto' GIT_COMMIT_SHA: $[ coalesce(variables['System.PullRequest.SourceCommitId'], variables['Build.SourceVersion'], 'not-found') ] + PIP_NO_WARN_SCRIPT_LOCATION: '0' + PYTEST_ADDOPTS: "-v -v -ra --showlocals" + PYTEST_XDIST_PROC_NR: 'auto' + trigger: batch: true @@ -30,7 +32,7 @@ trigger: jobs: - job: notify_build_start - pool: {vmImage: 'Ubuntu 16.04'} + pool: {vmImage: "Ubuntu-16.04"} steps: - script: | printenv && \ @@ -79,7 +81,7 @@ jobs: parameters: {tox: py27, python: 2.7, os: macOs} - job: report_coverage - pool: {vmImage: 'Ubuntu 16.04'} + pool: {vmImage: "Ubuntu-16.04"} condition: always() dependsOn: - windows_py37 @@ -139,9 +141,8 @@ jobs: - task: PublishCodeCoverageResults@1 displayName: publish overall coverage report to Azure inputs: - codeCoverageTool: 'cobertura' - summaryFileLocation: '$(System.DefaultWorkingDirectory)/.tox/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/.tox/htmlcov' + codeCoverageTool: 'Cobertura' + summaryFileLocation: "$(System.DefaultWorkingDirectory)/.tox/coverage.xml" failIfCoverageEmpty: true - script: | @@ -184,7 +185,7 @@ jobs: - linux_docs - linux_package_description condition: succeeded() - pool: {vmImage: 'Ubuntu 16.04'} + pool: {vmImage: "Ubuntu-16.04"} steps: - task: UsePythonVersion@0 displayName: setup python3.7 diff --git a/azure-run-tox-env.yml b/azure-run-tox-env.yml index 3f0c7cbba..449f03a7e 100644 --- a/azure-run-tox-env.yml +++ b/azure-run-tox-env.yml @@ -16,12 +16,12 @@ jobs: variables: TMPDIR: $(Build.BinariesDirectory) + ${{ if in(parameters.tox, 'pypy', 'py27', 'py34') }}: + PYTHONWARNINGS: 'ignore:::pip._internal.cli.base_command' ${{ if in(parameters.python, 'pypy', 'pypy3') }}: python: ${{ parameters.python }} ${{ if notIn(parameters.python, 'pypy', 'pypy3') }}: python: "python" - ${{ if in(parameters.python, 'pypy', 'py27', 'py34') }}: - PYTHONWARNINGS: 'ignore:::pip._internal.cli.base_command' steps: # ensure the required Python versions are available @@ -31,7 +31,7 @@ jobs: inputs: versionSpec: ${{ parameters.python }} - - script: "$(python) -c \"import sys; print(sys.version); print(sys.executable)\"" + - script: 'python -c "import sys; print(sys.version); print(sys.executable);"' displayName: show python information - script: "python -m pip install -U pip setuptools --user" diff --git a/setup.cfg b/setup.cfg index 202f79cb6..d9d0759e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,6 @@ testing = pytest >= 3.0.0, <5 pytest-cov >= 2.5.1, <3 pytest-mock >= 1.10.0, <2 - pytest-timeout >= 1.3.0, <2 pytest-xdist >= 1.22.2, <2 pytest-randomly >= 1.2.3, <2 psutil >= 5.6.1, < 6; python_version != "3.4" diff --git a/tox.ini b/tox.ini index c800093fa..4ce52b457 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,6 @@ extras = testing commands = pytest \ --cov "{envsitepackagesdir}/tox" \ --cov-config "{toxinidir}/tox.ini" \ - --timeout 180 \ --junitxml {env:JUNIT_XML_FILE:{toxworkdir}/.test.{envname}.xml} \ -n={env:PYTEST_XDIST_PROC_NR:auto} \ {posargs:.} From 3c6b4f204e89852c4b7536b246a66d20be6d39ec Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 17 Apr 2019 15:08:01 +0200 Subject: [PATCH 05/72] =?UTF-8?q?=F0=9F=94=A5=20Cut=20comments=20off=20dep?= =?UTF-8?q?endency=20lines=20(#1262)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enables users to have inline comments in deps lines in `tox.ini`. In particular, this allows having pyup bot integration. Ref: https://github.com/pyupio/dparse/issues/34 Ref: https://github.com/ansible/molecule/issues/1973 Fixes #1260 --- docs/changelog/1262.feature.rst | 1 + docs/config.rst | 69 +++++++++++--------- docs/example/package.rst | 13 ++-- docs/links.rst | 7 ++ setup.cfg | 3 +- src/tox/config/__init__.py | 5 ++ tests/integration/test_parallel_interrupt.py | 2 + tests/unit/config/test_config.py | 4 ++ tests/unit/package/test_package_parallel.py | 7 +- tox.ini | 4 +- 10 files changed, 69 insertions(+), 46 deletions(-) create mode 100644 docs/changelog/1262.feature.rst diff --git a/docs/changelog/1262.feature.rst b/docs/changelog/1262.feature.rst new file mode 100644 index 000000000..63a86f033 --- /dev/null +++ b/docs/changelog/1262.feature.rst @@ -0,0 +1 @@ +Allow having inline comments in :conf:`deps` — by :user:`webknjaz` diff --git a/docs/config.rst b/docs/config.rst index c71eac142..90b3d5cb7 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -21,9 +21,6 @@ the *ini-style* format under the ``tool.tox.legacy_tox_ini`` key as a multi-line Below you find the specification for the *ini-style* format, but you might want to skim some :doc:`examples` first and use this page as a reference. -.. _ConfigParser: https://docs.python.org/3/library/configparser.html - - tox global settings ------------------- @@ -45,10 +42,10 @@ Global settings are defined under the ``tox`` section as: .. versionadded:: 3.2.0 Specify python packages that need to exist alongside the tox installation for the tox build - to be able to start. Use this to specify plugin requirements (or the version of ``virtualenv`` - - determines the default ``pip``, ``setuptools``, and ``wheel`` versions the tox environments - start with). If these dependencies are not specified tox will create :conf:`provision_tox_env` - environment so that they are satisfied and delegate all calls to that. + to be able to start (must be PEP-508_ compliant). Use this to specify plugin requirements + (or the version of ``virtualenv`` - determines the default ``pip``, ``setuptools``, and ``wheel`` + versions the tox environments start with). If these dependencies are not specified tox will create + :conf:`provision_tox_env` environment so that they are satisfied and delegate all calls to that. .. code-block:: ini @@ -152,10 +149,8 @@ Global settings are defined under the ``tox`` section as: Activate isolated build environment. tox will use a virtual environment to build a source distribution from the source tree. For build tools and arguments use - the ``pyproject.toml`` file as specified in - `PEP-517 `_ and - `PEP-518 `_. To specify the virtual - environment Python version define use the :conf:`isolated_build_env` config + the ``pyproject.toml`` file as specified in `PEP-517`_ and `PEP-518`_. To specify the + virtual environment Python version define use the :conf:`isolated_build_env` config section. .. conf:: isolated_build_env ^ string ^ .package @@ -339,26 +334,32 @@ Complete list of settings that you can put into ``testenv*`` sections: .. conf:: deps ^ MULTI-LINE-LIST - Test-specific dependencies - to be installed into the environment prior to project - package installation. Each line defines a dependency, which will be - passed to the installer command for processing (see :conf:`indexserver`). - Each line specifies a file, a URL or a package name. You can additionally specify - an :conf:`indexserver` to use for installing this dependency - but this functionality is deprecated since tox-2.3. - All derived dependencies (deps required by the dep) will then be - retrieved from the specified indexserver: + Environment dependencies - installed into the environment ((see :conf:`install_command`) prior + to project after environment creation. One dependency (a file, a URL or a package name) per + line. Must be PEP-508_ compliant. All installer commands are executed using the toxinidir_ as the + current working directory. .. code-block:: ini - [tox] - indexserver = - myindexserver = https://myindexserver.example.com/simple - [testenv] - deps = :myindexserver:pkg + deps = + pytest + pytest-cov >= 3.5 + pywin32 >=1.0 ; sys_platform == 'win32' + octomachinery==0.0.13 # pyup: < 0.1.0 # disable feature updates + - (Experimentally introduced in 1.6.1) all installer commands are executed - using the ``{toxinidir}`` as the current working directory. + .. versionchanged:: 2.3 + + Support for index servers is now deprecated, and it's usage discouraged. + + .. versionchanged:: 3.9 + + Comment support on the same line as the dependency. When feeding the content to the install + tool we'll strip off content (including) from the first comment marker (``#``) + preceded by one or more space. For example, if a dependency is + ``octomachinery==0.0.13 # pyup: < 0.1.0 # disable feature updates`` it will be turned into + just ``octomachinery==0.0.13``. .. conf:: platform ^ REGEX @@ -559,8 +560,12 @@ having value magic). Globally available substitutions ++++++++++++++++++++++++++++++++ +.. _`toxinidir`: + ``{toxinidir}`` - the directory where tox.ini is located + the directory where ``tox.ini`` is located + +.. _`toxworkdir`: ``{toxworkdir}`` the directory where virtual environments are created and sub directories @@ -846,11 +851,11 @@ special case for a combination of factors. Here is how you do it: [testenv] deps = - py34-mysql: PyMySQL ; use if both py34 and mysql are in the env name - py27,py36: urllib3 ; use if either py36 or py27 are in the env name - py{27,36}-sqlite: mock ; mocking sqlite in python 2.x & 3.6 - !py34-sqlite: mock ; mocking sqlite, except in python 3.4 - sqlite-!py34: mock ; (same as the line above) + py34-mysql: PyMySQL # use if both py34 and mysql are in the env name + py27,py36: urllib3 # use if either py36 or py27 are in the env name + py{27,36}-sqlite: mock # mocking sqlite in python 2.x & 3.6 + !py34-sqlite: mock # mocking sqlite, except in python 3.4 + sqlite-!py34: mock # (same as the line above) Take a look at the first ``deps`` line. It shows how you can special case something for a combination of factors, by just hyphenating the combining diff --git a/docs/example/package.rst b/docs/example/package.rst index 642dd4832..b0c38b254 100644 --- a/docs/example/package.rst +++ b/docs/example/package.rst @@ -4,13 +4,12 @@ packaging Although one can use tox to develop and test applications one of its most popular usage is to help library creators. Libraries need first to be packaged, so then they can be installed inside a virtual environment for testing. To help with this -tox implements `PEP-517 `_ and -`PEP-518 `_. This means that by default +tox implements PEP-517_ and PEP-518_. This means that by default tox will build source distribution out of source trees. Before running test commands ``pip`` is used to install the source distribution inside the build environment. -To create a source distribution there are multiple tools out there and with ``PEP-517`` -and ``PEP-518`` you can easily use your favorite one with tox. Historically tox +To create a source distribution there are multiple tools out there and with PEP-517_ and PEP-518_ +you can easily use your favorite one with tox. Historically tox only supported ``setuptools``, and always used the tox host environment to build a source distribution from the source tree. This is still the default behavior. To opt out of this behaviour you need to set isolated builds to true. @@ -37,7 +36,7 @@ build requirements. flit ---- -`flit `_ requires ``Python 3``, however the generated source +flit_ requires ``Python 3``, however the generated source distribution can be installed under ``python 2``. Furthermore it does not require a ``setup.py`` file as that information is also added to the ``pyproject.toml`` file. @@ -68,7 +67,7 @@ file as that information is also added to the ``pyproject.toml`` file. poetry ------ -`poetry `_ requires ``Python 3``, however the generated source +poetry_ requires ``Python 3``, however the generated source distribution can be installed under ``python 2``. Furthermore it does not require a ``setup.py`` file as that information is also added to the ``pyproject.toml`` file. @@ -95,3 +94,5 @@ file as that information is also added to the ``pyproject.toml`` file. # so unless this is python 3 you can require a given python version for the packaging # environment via the basepython key basepython = python3 + +.. include:: ../links.rst diff --git a/docs/links.rst b/docs/links.rst index 97789f416..ba75c6022 100644 --- a/docs/links.rst +++ b/docs/links.rst @@ -9,6 +9,7 @@ .. _`nose`: https://pypi.org/project/nose .. _`Holger Krekel`: https://twitter.com/hpk42 .. _`pytest-xdist`: https://pypi.org/project/pytest-xdist +.. _ConfigParser: https://docs.python.org/3/library/configparser.html .. _`easy_install`: http://peak.telecommunity.com/DevCenter/EasyInstall .. _pip: https://pypi.org/project/pip @@ -18,7 +19,13 @@ .. _discover: https://pypi.org/project/discover .. _unittest2: https://pypi.org/project/unittest2 .. _mock: https://pypi.org/project/mock/ +.. _flit: https://flit.readthedocs.io/en/latest/ +.. _poetry: https://poetry.eustace.io/ .. _pypy: https://pypy.org .. _`Python Packaging Guide`: https://packaging.python.org/tutorials/packaging-projects/ .. _`tox.ini`: :doc:configfile + +.. _`PEP-508`: https://www.python.org/dev/peps/pep-0508/ +.. _`PEP-517`: https://www.python.org/dev/peps/pep-0517/ +.. _`PEP-518`: https://www.python.org/dev/peps/pep-0518/ diff --git a/setup.cfg b/setup.cfg index d9d0759e1..c5726a8bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,8 +64,9 @@ testing = pytest-xdist >= 1.22.2, <2 pytest-randomly >= 1.2.3, <2 psutil >= 5.6.1, < 6; python_version != "3.4" + flaky >= 3.4.0, < 4 docs = - sphinx >= 1.8.0, < 2 + sphinx >= 2.0.0, < 3 towncrier >= 18.5.0 pygments-github-lexers >= 0.0.5 sphinxcontrib-autoprogram >= 0.1.5 diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index ea6d96763..8180705b4 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -158,6 +158,7 @@ def postprocess(self, testenv_config, value): name_start = "{} ".format(option) if name.startswith(name_start): name = "{}={}".format(option, name[len(option) :].strip()) + name = self._cut_off_dep_comment(name) name = self._replace_forced_dep(name, config) deps.append(DepConfig(name, ixserver)) return deps @@ -176,6 +177,10 @@ def _replace_forced_dep(self, name, config): return forced_dep return name + @staticmethod + def _cut_off_dep_comment(name): + return re.sub(r"\s+#.*", "", name).strip() + @classmethod def _is_same_dep(cls, dep1, dep2): """Definitions are the same if they refer to the same package, even if versions differ.""" diff --git a/tests/integration/test_parallel_interrupt.py b/tests/integration/test_parallel_interrupt.py index 496579a33..028906e09 100644 --- a/tests/integration/test_parallel_interrupt.py +++ b/tests/integration/test_parallel_interrupt.py @@ -6,11 +6,13 @@ from datetime import datetime import pytest +from flaky import flaky from pathlib2 import Path from tox.util.main import MAIN_FILE +@flaky(max_runs=3) @pytest.mark.skipif( "sys.platform == 'win32'", reason="triggering SIGINT reliably on Windows is hard" ) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index ccb8200f0..d46f58640 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -142,7 +142,9 @@ def test_process_deps(self, newconfig): [testenv] deps = -r requirements.txt + yapf>=0.25.0,<0.27 # pyup: < 0.27 # disable updates --index-url https://pypi.org/simple + pywin32 >=1.0 ; sys_platform == '#my-magic-platform' # so what now -fhttps://pypi.org/packages --global-option=foo -v dep1 @@ -151,7 +153,9 @@ def test_process_deps(self, newconfig): ) # note that those last two are invalid expected_deps = [ "-rrequirements.txt", + "yapf>=0.25.0,<0.27", "--index-url=https://pypi.org/simple", + "pywin32 >=1.0 ; sys_platform == '#my-magic-platform'", "-fhttps://pypi.org/packages", "--global-option=foo", "-v dep1", diff --git a/tests/unit/package/test_package_parallel.py b/tests/unit/package/test_package_parallel.py index fa376164a..70a5b4d1a 100644 --- a/tests/unit/package/test_package_parallel.py +++ b/tests/unit/package/test_package_parallel.py @@ -1,16 +1,13 @@ import os -import platform import traceback import py -import pytest +from flaky import flaky from tox.session.commands.run import sequential -@pytest.mark.skipif( - platform.python_implementation().lower() == "pypy", reason="this is flaky on pypy" -) +@flaky(max_runs=3) def test_tox_parallel_build_safe(initproj, cmd, mock_venv, monkeypatch): initproj( "env_var_test", diff --git a/tox.ini b/tox.ini index 4ce52b457..fe777146b 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ source = src/tox *\src\tox [pytest] -addopts = -ra --showlocals +addopts = -ra --showlocals --no-success-flaky-report rsyncdirs = tests tox looponfailroots = tox tests testpaths = tests @@ -136,7 +136,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,docutils,filelock,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml +known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml [testenv:release] description = do a release, required posarg of the version number From dbd889b4728c6f09ba84c131286642e09b16a190 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Wed, 17 Apr 2019 14:29:19 +0100 Subject: [PATCH 06/72] release 3.9.0 --- docs/changelog.rst | 17 +++++++++++++++++ docs/changelog/1257.bugfix.rst | 1 - docs/changelog/1262.feature.rst | 1 - 3 files changed, 17 insertions(+), 2 deletions(-) delete mode 100644 docs/changelog/1257.bugfix.rst delete mode 100644 docs/changelog/1262.feature.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 082b70576..f45536d6d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,23 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.9.0 (2019-04-17) +------------------- + +Bugfixes +^^^^^^^^ + +- Fix ``congratulations`` when using ``^C`` during virtualenv creation - by :user:`asottile` + `#1257 `_ + + +Features +^^^^^^^^ + +- Allow having inline comments in :conf:`deps` — by :user:`webknjaz` + `#1262 `_ + + v3.8.6 (2019-04-03) ------------------- diff --git a/docs/changelog/1257.bugfix.rst b/docs/changelog/1257.bugfix.rst deleted file mode 100644 index f18ff223d..000000000 --- a/docs/changelog/1257.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``congratulations`` when using ``^C`` during virtualenv creation - by :user:`asottile` diff --git a/docs/changelog/1262.feature.rst b/docs/changelog/1262.feature.rst deleted file mode 100644 index 63a86f033..000000000 --- a/docs/changelog/1262.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Allow having inline comments in :conf:`deps` — by :user:`webknjaz` From 8bf4d9bffd63e8e885f362e8f00f48ba46767cde Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 22 Apr 2019 20:40:31 -0700 Subject: [PATCH 07/72] Trim passenv (tox>=2.8) (#1269) Committed via https://github.com/asottile/all-repos --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index fe777146b..54ce721b1 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,6 @@ commands = pip wheel -w {envtmpdir}/build --no-deps . description = format the code base to adhere to our styles, and complain about what we cannot do automatically basepython = python3.7 passenv = {[testenv]passenv} - HOMEPATH # without PROGRAMDATA cloning using git for Windows will fail with an # `error setting certificate verify locations` error PROGRAMDATA From c1e4d54dc78fa156f473ecb1e3911943641c793f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 23 Apr 2019 10:14:43 +0100 Subject: [PATCH 08/72] Update bug_report.md --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index d3157a58a..7387865e4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Something that does not works as expected -title: "[BUG] " +title: "" labels: bug:normal assignees: '' From a338734b3478c62ca87ad3f511b5fdf57c1c6ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 23 Apr 2019 10:15:00 +0100 Subject: [PATCH 09/72] Update feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index e5c08ffea..e3cf4e465 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an improvement for the project -title: "[FEATURE]" +title: "" labels: feature:new assignees: '' From 58a47582ed4961b7df6b1b4d82e41dc42bbe4b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 30 Apr 2019 07:57:29 +0100 Subject: [PATCH 10/72] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 00d1236bd..7e643fa92 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ PyPi](https://badge.fury.io/py/tox.svg)](https://badge.fury.io/py/tox) versions](https://img.shields.io/pypi/pyversions/tox.svg)](https://pypi.org/project/tox/) [![Azure Pipelines build status](https://dev.azure.com/toxdev/tox/_apis/build/status/tox%20ci?branchName=master)](https://dev.azure.com/toxdev/tox/_build/latest?definitionId=9&branchName=master) -[![Test -Coverage](https://api.codeclimate.com/v1/badges/425c19ab2169a35e1c16/test_coverage)](https://codeclimate.com/github/tox-dev/tox/code?sort=test_coverage) [![Documentation status](https://readthedocs.org/projects/tox/badge/?version=latest&style=flat-square)](https://tox.readthedocs.io/en/latest/?badge=latest) [![Code style: From 12eb2b6a82960508de285289657ef6658b323d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 1 May 2019 05:52:44 -0400 Subject: [PATCH 11/72] allow passing unknown args to provisioned tox (#1282) tox ignores unknown CLI arguments when provisioning is on and outside of the provisioned environment (allowing provisioning arguments to be forwarded freely) --- docs/changelog/1270.bugfix.rst | 2 ++ src/tox/config/__init__.py | 12 +++++-- src/tox/session/commands/provision.py | 6 +++- tests/unit/__init__.py | 0 tests/unit/session/plugin/a/__init__.py | 8 +++++ tests/unit/session/plugin/setup.cfg | 10 ++++++ tests/unit/session/plugin/setup.py | 3 ++ tests/unit/session/test_provision.py | 47 +++++++++++++++++++++++++ 8 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/1270.bugfix.rst create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/session/plugin/a/__init__.py create mode 100644 tests/unit/session/plugin/setup.cfg create mode 100644 tests/unit/session/plugin/setup.py diff --git a/docs/changelog/1270.bugfix.rst b/docs/changelog/1270.bugfix.rst new file mode 100644 index 000000000..38afe0661 --- /dev/null +++ b/docs/changelog/1270.bugfix.rst @@ -0,0 +1,2 @@ +tox ignores unknown CLI arguments when provisioning is on and outside of the provisioned environment (allowing +provisioning arguments to be forwarded freely) - by :user:`gaborbernat` diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 8180705b4..70aa45a3b 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -45,6 +45,8 @@ default_factors = tox.PYTHON.DEFAULT_FACTORS """DEPRECATED MOVE - please update to new location.""" +WITHIN_PROVISION = os.environ.get(str("TOX_PROVISION")) == "1" + def get_plugin_manager(plugins=()): # initialize plugin manager @@ -114,8 +116,11 @@ def add_testenv_attribute_obj(self, obj): assert hasattr(obj, "postprocess") self._testenv_attr.append(obj) - def parse_cli(self, args): - return self.argparser.parse_args(args) + def parse_cli(self, args, strict=False): + if strict or WITHIN_PROVISION: + return self.argparser.parse_args(args) + else: + return self.argparser.parse_known_args(args)[0] def _format_help(self): return self.argparser.format_help() @@ -1106,6 +1111,9 @@ def handle_provision(self, config, reader): env_config.deps = deps config.envconfigs[config.provision_tox_env] = env_config raise tox.exception.MissingRequirement(config) + # if provisioning is not on, now we need do a strict argument evaluation + # raise on unknown args + self.config._parser.parse_cli(args=self.config.args, strict=True) @staticmethod def ensure_requires_satisfied(config, requires, min_version): diff --git a/src/tox/session/commands/provision.py b/src/tox/session/commands/provision.py index 3f0ac6196..21825fc22 100644 --- a/src/tox/session/commands/provision.py +++ b/src/tox/session/commands/provision.py @@ -1,6 +1,8 @@ """In case the tox environment is not correctly setup provision it and delegate execution""" from __future__ import absolute_import, unicode_literals +import os + from tox.exception import InvocationError @@ -9,7 +11,9 @@ def provision_tox(provision_venv, args): with provision_venv.new_action("provision") as action: provision_args = [str(provision_venv.envconfig.envpython), "-m", "tox"] + args try: - action.popen(provision_args, redirect=False, report_fail=False) + env = os.environ.copy() + env[str("TOX_PROVISION")] = str("1") + action.popen(provision_args, redirect=False, report_fail=False, env=env) return 0 except InvocationError as exception: return exception.exit_code diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/session/plugin/a/__init__.py b/tests/unit/session/plugin/a/__init__.py new file mode 100644 index 000000000..dbe246397 --- /dev/null +++ b/tests/unit/session/plugin/a/__init__.py @@ -0,0 +1,8 @@ +import pluggy + +hookimpl = pluggy.HookimplMarker("tox") + + +@hookimpl +def tox_addoption(parser): + parser.add_argument("--option", choices=["a", "b"], default="a", required=False) diff --git a/tests/unit/session/plugin/setup.cfg b/tests/unit/session/plugin/setup.cfg new file mode 100644 index 000000000..285b17de5 --- /dev/null +++ b/tests/unit/session/plugin/setup.cfg @@ -0,0 +1,10 @@ +[metadata] +name = plugin +description = test stuff +version = 0.1 +[options] +zip_safe = True +packages = find: + +[options.entry_points] +tox = plugin = a diff --git a/tests/unit/session/plugin/setup.py b/tests/unit/session/plugin/setup.py new file mode 100644 index 000000000..606849326 --- /dev/null +++ b/tests/unit/session/plugin/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() diff --git a/tests/unit/session/test_provision.py b/tests/unit/session/test_provision.py index ae4b4abc7..6408dc654 100644 --- a/tests/unit/session/test_provision.py +++ b/tests/unit/session/test_provision.py @@ -1,5 +1,8 @@ +import shutil +import subprocess import sys +import py import pytest from tox.exception import BadRequirement, MissingRequirement @@ -94,3 +97,47 @@ def test_provision_bad_requires(newconfig, capsys, monkeypatch): out, err = capsys.readouterr() assert "ERROR: failed to parse RequirementParseError" in out assert not err + + +@pytest.fixture() +def plugin(monkeypatch, tmp_path): + dest = tmp_path / "a" + shutil.copytree(str(py.path.local(__file__).dirpath().join("plugin")), str(dest)) + subprocess.check_output([sys.executable, "setup.py", "egg_info"], cwd=str(dest)) + monkeypatch.setenv(str("PYTHONPATH"), str(dest)) + + +def test_provision_cli_args_ignore(cmd, initproj, monkeypatch, plugin): + import tox.config + import tox.session + + prev_ensure = tox.config.ParseIni.ensure_requires_satisfied + + @staticmethod + def ensure_requires_satisfied(config, requires, min_version): + result = prev_ensure(config, requires, min_version) + config.run_provision = True + return result + + monkeypatch.setattr( + tox.config.ParseIni, "ensure_requires_satisfied", ensure_requires_satisfied + ) + prev_get_venv = tox.session.Session.getvenv + + def getvenv(self, name): + venv = prev_get_venv(self, name) + venv.envconfig.envdir = py.path.local(sys.executable).dirpath().dirpath() + venv.setupenv = lambda: True + venv.finishvenv = lambda: True + return venv + + monkeypatch.setattr(tox.session.Session, "getvenv", getvenv) + initproj("test-0.1", {"tox.ini": "[tox]"}) + result = cmd("-a", "--option", "b") + result.assert_success(is_run_test_env=False) + + +def test_provision_cli_args_not_ignored_if_provision_false(cmd, initproj): + initproj("test-0.1", {"tox.ini": "[tox]"}) + result = cmd("-a", "--option", "b") + result.assert_fail(is_run_test_env=False) From 7a2edd942602a4ad9b0efd16b8f92e3d7b16f4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 1 May 2019 05:52:59 -0400 Subject: [PATCH 12/72] do not allow setting the ``TOXENV`` or the ``-e` flag to override the listed default environment variables, these show up under extra (#1284) --- docs/changelog/720.bugfix.rst | 1 + src/tox/config/__init__.py | 4 ++-- src/tox/session/commands/show_env.py | 2 +- tests/unit/session/test_list_env.py | 25 ++++++++++++++++++++++++- 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/720.bugfix.rst diff --git a/docs/changelog/720.bugfix.rst b/docs/changelog/720.bugfix.rst new file mode 100644 index 000000000..6b17d8a70 --- /dev/null +++ b/docs/changelog/720.bugfix.rst @@ -0,0 +1 @@ +fix for ``tox -l`` command: do not allow setting the ``TOXENV`` or the ``-e`` flag to override the listed default environment variables, they still show up under extra if non defined target - by :user:`gaborbernat` diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 70aa45a3b..adfa81fbd 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -1063,7 +1063,7 @@ def line_of_default_to_zero(section, name=None): self.handle_provision(config, reader) self.parse_build_isolation(config, reader) - config.envlist, all_envs = self._getenvdata(reader, config) + config.envlist, all_envs, config.envlist_default = self._getenvdata(reader, config) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -1251,7 +1251,7 @@ def _getenvdata(self, reader, config): if config.isolated_build is True and package_env in env_list: msg = "isolated_build_env {} cannot be part of envlist".format(package_env) raise tox.exception.ConfigError(msg) - return env_list, all_envs + return env_list, all_envs, _split_env(from_config) def _split_env(env): diff --git a/src/tox/session/commands/show_env.py b/src/tox/session/commands/show_env.py index f741234a0..72e49db3e 100644 --- a/src/tox/session/commands/show_env.py +++ b/src/tox/session/commands/show_env.py @@ -5,7 +5,7 @@ def show_envs(config, all_envs=False, description=False): env_conf = config.envconfigs # this contains all environments - default = config.envlist # this only the defaults + default = config.envlist_default # this only the defaults ignore = {config.isolated_build_env, config.provision_tox_env}.union(default) extra = [e for e in env_conf if e not in ignore] if all_envs else [] diff --git a/tests/unit/session/test_list_env.py b/tests/unit/session/test_list_env.py index 364049033..8281a8296 100644 --- a/tests/unit/session/test_list_env.py +++ b/tests/unit/session/test_list_env.py @@ -20,6 +20,18 @@ def test_listenvs(cmd, initproj, monkeypatch): """ }, ) + + result = cmd("-l") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] + + result = cmd("-l", "-e", "py") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] + + monkeypatch.setenv(str("TOXENV"), str("py")) + result = cmd("-l") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] + + monkeypatch.setenv(str("TOXENV"), str("py36")) result = cmd("-l") assert result.outlines == ["py36", "py27", "py34", "pypi", "docs"] @@ -60,7 +72,7 @@ def test_listenvs_verbose_description(cmd, initproj): assert result.outlines[2:] == expected -def test_listenvs_all(cmd, initproj): +def test_listenvs_all(cmd, initproj, monkeypatch): initproj( "listenvs_all", filedefs={ @@ -80,6 +92,17 @@ def test_listenvs_all(cmd, initproj): expected = ["py36", "py27", "py34", "pypi", "docs", "notincluded"] assert result.outlines == expected + result = cmd("-a", "-e", "py") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs", "py", "notincluded"] + + monkeypatch.setenv(str("TOXENV"), str("py")) + result = cmd("-a") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs", "py", "notincluded"] + + monkeypatch.setenv(str("TOXENV"), str("py36")) + result = cmd("-a") + assert result.outlines == ["py36", "py27", "py34", "pypi", "docs", "notincluded"] + def test_listenvs_all_verbose_description(cmd, initproj): initproj( From 18147f4c5af75caf40c3c2f95a0907da2db83deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 1 May 2019 05:53:17 -0400 Subject: [PATCH 13/72] add no download (#1283) --- docs/changelog/448.feature.rst | 3 +++ setup.cfg | 2 +- src/tox/config/__init__.py | 8 ++++++++ src/tox/venv.py | 3 +-- tests/conftest.py | 3 --- tests/unit/test_venv.py | 25 +++++++++++++++++++++++++ tox.ini | 1 + 7 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 docs/changelog/448.feature.rst diff --git a/docs/changelog/448.feature.rst b/docs/changelog/448.feature.rst new file mode 100644 index 000000000..02d87e13c --- /dev/null +++ b/docs/changelog/448.feature.rst @@ -0,0 +1,3 @@ +Virtual environments created now no longer upgrade pip/wheel/setuptools to the latest version. Instead the start +packages after virtualenv creation now is whatever virtualenv has bundled in. This allows faster virtualenv +creation and builds that are easier to reproduce. diff --git a/setup.cfg b/setup.cfg index c5726a8bc..569f4ca2e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = pluggy >= 0.3.0, <1 py >= 1.4.17, <2 six >= 1.0.0, <2 - virtualenv >= 1.11.2 + virtualenv >= 14.0.0 toml >=0.9.4 filelock >= 3.0.0, <4 diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index adfa81fbd..574dc9526 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -749,6 +749,14 @@ def alwayscopy(testenv_config, value): "have access to globally installed packages.", ) + parser.add_testenv_attribute( + "download", + type="bool", + default=False, + help="download the latest pip, setuptools and wheel when creating the virtual" + "environment (default is to use the one bundled in virtualenv)", + ) + parser.add_testenv_attribute( name="alwayscopy", type="bool", diff --git a/src/tox/venv.py b/src/tox/venv.py index 22e7e7ae4..1f7860433 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -646,7 +646,6 @@ def prepend_shebang_interpreter(args): return args -NO_DOWNLOAD = False _SKIP_VENV_CREATION = os.environ.get("_TOX_SKIP_ENV_CREATION_TEST", False) == "1" @@ -658,7 +657,7 @@ def tox_testenv_create(venv, action): args.append("--system-site-packages") if venv.envconfig.alwayscopy: args.append("--always-copy") - if NO_DOWNLOAD: + if not venv.envconfig.download: args.append("--no-download") # add interpreter explicitly, to prevent using default (virtualenv.ini) args.extend(["--python", str(config_interpreter)]) diff --git a/tests/conftest.py b/tests/conftest.py index cf0821a12..ec59f4a1c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,4 @@ # TODO move fixtures here and only keep helper functions/classes in the plugin # TODO _pytest_helpers might be a better name than _pytestplugin then? # noinspection PyUnresolvedReferences -import tox.venv from tox._pytestplugin import * # noqa - -tox.venv.NO_DOWNLOAD = True diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index da6298e43..18dafb889 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -1052,3 +1052,28 @@ def test_tox_testenv_interpret_shebang_long_example(tmpdir): ] assert args == expected + base_args + + +@pytest.mark.parametrize("download", [True, False, None]) +def test_create_download(mocksession, newconfig, download): + config = newconfig( + [], + """\ + [testenv:env] + {} + """.format( + "download={}".format(download) if download else "" + ), + ) + mocksession.new_config(config) + venv = mocksession.getvenv("env") + with mocksession.newaction(venv.name, "getenv") as action: + tox_testenv_create(action=action, venv=venv) + pcalls = mocksession._pcalls + assert len(pcalls) >= 1 + args = pcalls[0].args + if download is True: + assert "--no-download" not in map(str, args) + else: + assert "--no-download" in map(str, args) + mocksession._clearmocks() diff --git a/tox.ini b/tox.ini index 54ce721b1..73c4af212 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ skip_missing_interpreters = true description = run the tests with pytest under {basepython} setenv = PIP_DISABLE_VERSION_CHECK = 1 COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} + VIRTUALENV_NO_DOWNLOAD = 1 passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_* deps = extras = testing From d33b803d5b17809d64644996eb4803890786a03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 1 May 2019 05:53:35 -0400 Subject: [PATCH 14/72] simplify CI/remove code climate (#1277) --- .codeclimate.yml | 55 ---------- azure-pipelines.yml | 249 ++++++++++-------------------------------- azure-run-tox-env.yml | 84 -------------- tox.ini | 2 +- 4 files changed, 60 insertions(+), 330 deletions(-) delete mode 100644 .codeclimate.yml delete mode 100644 azure-run-tox-env.yml diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 4fa9ccbee..000000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: "2" -checks: - argument-count: - config: - threshold: 6 - complex-logic: - config: - threshold: 10 - file-lines: - config: - threshold: 500 - method-complexity: - config: - threshold: 10 - method-count: - config: - threshold: 20 - method-lines: - config: - threshold: 50 - nested-control-flow: - config: - threshold: 4 - return-statements: - config: - threshold: 10 - -plugins: - sonar-python: - enabled: true - config: - tests_patterns: - - tests/** - pep8: - enabled: false - radon: - enabled: false - -exclude_patterns: - - "docs/" - - "tasks/" - - ".github/" - - ".pre-commit-config.yml" - - "*.yml" - - "*.yaml" - - "*.rst" - - "*.ini" - - ".gitignore" - - ".codeclimate.yml" - - "CONTRIBUTORS" - - "CODE_OF_CONDUCT.md" - - "LICENSE" - - "MANIFEST.in" - - "pyproject.toml" - - "setup.cfg" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6dda98580..279df065f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,16 +1,12 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -variables: - "System.PreferGit": true - CI_NAME: Azure Pipelines - CI_BUILD_ID: $(Build.BuildId) - CI_BUILD_URL: "https://toxdev.visualstudio.com/tox/_build/results?buildId=$(Build.BuildId)" - GIT_BRANCH: $[ coalesce(variables['System.PullRequest.SourceBranch'], variables['Build.SourceBranchName'], 'not-found') ] - GIT_COMMIT_SHA: $[ coalesce(variables['System.PullRequest.SourceCommitId'], variables['Build.SourceVersion'], 'not-found') ] - PIP_NO_WARN_SCRIPT_LOCATION: '0' - PYTEST_ADDOPTS: "-v -v -ra --showlocals" - PYTEST_XDIST_PROC_NR: 'auto' - +resources: + repositories: + - repository: tox + type: github + endpoint: toxdevorg + name: tox-dev/azure-pipelines-template + ref: master trigger: batch: true @@ -18,186 +14,59 @@ trigger: include: - master - refs/tags/* - paths: - exclude: - - readthedocs.yml - - LICENSE - - HOWTORELEASE.rst - - CONTRIBUTORS - - CONTRIBUTING.rst - - CODE_OF_CONDUCT.md - - .gitignore - - .github/* - - tasks/* - -jobs: -- job: notify_build_start - pool: {vmImage: "Ubuntu-16.04"} - steps: - - script: | - printenv && \ - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && \ - chmod +x ./cc-test-reporter && \ - ./cc-test-reporter before-build -d - displayName: notify code climate of new build - -- template: azure-run-tox-env.yml - parameters: {tox: fix_lint, python: 3.7} -- template: azure-run-tox-env.yml - parameters: {tox: docs, python: 3.7} -- template: azure-run-tox-env.yml - parameters: {tox: package_description, python: 3.7} - -- template: azure-run-tox-env.yml - parameters: {tox: pypy, python: pypy, os: linux} -- template: azure-run-tox-env.yml - parameters: {tox: pypy3, python: pypy3, os: linux} - -- template: azure-run-tox-env.yml - parameters: {tox: py37, python: 3.7, os: windows} -- template: azure-run-tox-env.yml - parameters: {tox: py36, python: 3.6, os: windows} -- template: azure-run-tox-env.yml - parameters: {tox: py35, python: 3.5, os: windows} -- template: azure-run-tox-env.yml - parameters: {tox: py34, python: 3.4, os: windows} -- template: azure-run-tox-env.yml - parameters: {tox: py27, python: 2.7, os: windows} - -- template: azure-run-tox-env.yml - parameters: {tox: py37, python: 3.7, os: linux} -- template: azure-run-tox-env.yml - parameters: {tox: py36, python: 3.6, os: linux} -- template: azure-run-tox-env.yml - parameters: {tox: py35, python: 3.5, os: linux} -- template: azure-run-tox-env.yml - parameters: {tox: py34, python: 3.4, os: linux} -- template: azure-run-tox-env.yml - parameters: {tox: py27, python: 2.7, os: linux} - -- template: azure-run-tox-env.yml - parameters: {tox: py37, python: 3.7, os: macOs} -- template: azure-run-tox-env.yml - parameters: {tox: py27, python: 2.7, os: macOs} - -- job: report_coverage - pool: {vmImage: "Ubuntu-16.04"} - condition: always() - dependsOn: - - windows_py37 - - windows_py36 - - windows_py35 - - windows_py34 - - windows_py27 - - linux_py37 - - linux_py36 - - linux_py35 - - linux_py34 - - linux_py27 - - linux_pypy3 - - linux_pypy - - macOS_py37 - - macOS_py27 - steps: - - task: DownloadBuildArtifacts@0 - displayName: download coverage files for run - inputs: - buildType: current - downloadType: specific - itemPattern: coverage-*/* - downloadPath: $(Build.StagingDirectory) - - - task: UsePythonVersion@0 - displayName: setup python - inputs: - versionSpec: 3.7 - - - script: | - python -c ' - from pathlib import Path - import shutil - - from_folder = Path("$(Build.StagingDirectory)") - destination_folder = Path("$(System.DefaultWorkingDirectory)") / ".tox" - destination_folder.mkdir() - for coverage_file in from_folder.glob("*/.coverage"): - destination = destination_folder / f".coverage.{coverage_file.parent.name[9:]}" - print(f"{coverage_file} copy to {destination}") - shutil.copy(str(coverage_file), str(destination))' - displayName: move coverage files into .tox - - - script: "python -m pip install -U pip setuptools --user" - displayName: upgrade pip - - - script: 'python -m pip install . -U --user' - displayName: install tox - eat our own dog food - - - script: 'python -m tox -e py --sdistonly' - displayName: generate version.py - - - script: 'python -m tox -e coverage' - displayName: create coverag report via tox - - - task: PublishCodeCoverageResults@1 - displayName: publish overall coverage report to Azure - inputs: - codeCoverageTool: 'Cobertura' - summaryFileLocation: "$(System.DefaultWorkingDirectory)/.tox/coverage.xml" - failIfCoverageEmpty: true +pr: + branches: + include: + - master - - script: | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && \ - chmod +x ./cc-test-reporter && \ - python -c ' - from xml.etree import ElementTree as et - from pathlib import Path - import subprocess +variables: + PYTEST_ADDOPTS: "-v -v -ra --showlocals" + PYTEST_XDIST_PROC_NR: 'auto' - from_folder = Path("$(Build.StagingDirectory)") - for counter, coverage_file in enumerate(from_folder.glob("*/coverage.xml")): - key = coverage_file.parent.name[9:] - print(f"{counter}) {coverage_file}") - try: - cmd = ["$(System.DefaultWorkingDirectory)/cc-test-reporter", "format-coverage", - str(coverage_file), - "-d", "-t", "coverage.py", - "-o", f"$(Build.StagingDirectory)/code-climate.{key}.json"] - print(f"\t{cmd}") - log = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) - code = 0 - except subprocess.CalledProcessError as exception: - log, code = exception.output, exception.returncode - finally: - print(code, log, "\n", sep="\n")' && \ - ./cc-test-reporter sum-coverage -d --output - \ - --parts $(ls -1 $(Build.StagingDirectory)/code-climate.*.json | wc -l) \ - $(Build.StagingDirectory)/code-climate.*.json | \ - ./cc-test-reporter -d -r d24f105984ab5e087773a21b8668acb0b36cb8311fc2637f78a2d9451e531e08 \ - upload-coverage --input - - displayName: publish code climate - condition: succeededOrFailed() +jobs: +- template: run-tox-env.yml@tox + parameters: + jobs: + check: + py: '3.7' + toxenvs: + - fix_lint + - docs + - package_description + windows: + coverage: 'coverage' + toxenvs: + - py37 + - py36 + - py35 + - py34 + - py27 + linux: + coverage: 'coverage' + toxenvs: + - py38 + - py37 + - py36 + - py35 + - py34 + - py27 + macOs: + coverage: 'coverage' + toxenvs: + - py37 + - py27 +- template: merge-coverage.yml@tox + parameters: + dependsOn: + - windows + - linux + - macOs - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}: - - job: publish - dependsOn: - - report_coverage - - linux_fix_lint - - linux_docs - - linux_package_description - condition: succeeded() - pool: {vmImage: "Ubuntu-16.04"} - steps: - - task: UsePythonVersion@0 - displayName: setup python3.7 - inputs: {versionSpec: '3.7'} - - task: TwineAuthenticate@0 - inputs: - externalFeeds: 'toxdev' - - script: 'python3.7 -m pip install -U twine pip . --user' - displayName: "acquire build tools" - - script: 'python3.7 -m pip wheel -w "$(System.DefaultWorkingDirectory)/w" --no-deps .' - displayName: "build wheel" - - script: 'python3.7 -m tox -e py --sdistonly' - displayName: "build sdist" - - script: 'python3.7 -m twine upload -r pypi-toxdev --config-file $(PYPIRC_PATH) $(System.DefaultWorkingDirectory)/w/* .tox/dist/*' - displayName: "upload sdist and wheel to PyPi" + - template: publish-pypi.yml@tox + parameters: + - external_feed: 'toxdev' + - pypi_remote: 'pypi-toxdev' + - dependsOn: + - check + - report_coverage diff --git a/azure-run-tox-env.yml b/azure-run-tox-env.yml deleted file mode 100644 index 449f03a7e..000000000 --- a/azure-run-tox-env.yml +++ /dev/null @@ -1,84 +0,0 @@ -parameters: - tox: "" - python: "" - os: "linux" - -jobs: -- job: ${{ format('{0}_{1}', parameters.os, parameters.tox) }} - dependsOn: notify_build_start - pool: - ${{ if eq(parameters.os, 'windows') }}: - vmImage: "windows-2019" - ${{ if eq(parameters.os, 'macOs') }}: - vmImage: "macOS-latest" - ${{ if eq(parameters.os, 'linux') }}: - vmImage: "Ubuntu-16.04" - - variables: - TMPDIR: $(Build.BinariesDirectory) - ${{ if in(parameters.tox, 'pypy', 'py27', 'py34') }}: - PYTHONWARNINGS: 'ignore:::pip._internal.cli.base_command' - ${{ if in(parameters.python, 'pypy', 'pypy3') }}: - python: ${{ parameters.python }} - ${{ if notIn(parameters.python, 'pypy', 'pypy3') }}: - python: "python" - - steps: - # ensure the required Python versions are available - - ${{ if notIn(parameters.python, 'pypy', 'pypy3') }}: - - task: UsePythonVersion@0 - displayName: setup python - inputs: - versionSpec: ${{ parameters.python }} - - - script: 'python -c "import sys; print(sys.version); print(sys.executable);"' - displayName: show python information - - - script: "python -m pip install -U pip setuptools --user" - displayName: upgrade pip - - - script: "python -m pip install -U . --user" - displayName: install tox - eat our own dog food - - - script: ${{ format('python -m tox -e {0} --notest', parameters.tox) }} - displayName: install test dependencies - - - ${{ if startsWith(parameters.tox, 'py') }}: - - script: python -m tox -e coverage --notest - displayName: install coverage dependencies - - - script: ${{ format('python -m tox -e {0}', parameters.tox) }} - displayName: run tests - - - ${{ if startsWith(parameters.tox, 'py') }}: - - task: PublishTestResults@2 - displayName: publish test results via junit - condition: succeededOrFailed() - inputs: - testResultsFormat: "JUnit" - testResultsFiles: ${{ format('$(System.DefaultWorkingDirectory)/.tox/.test.{0}.xml', parameters.tox) }} - testRunTitle: ${{ format('{0}_{1}', parameters.os, parameters.tox) }} - - - ${{ if startsWith(parameters.tox, 'py') }}: - - script: "python -m tox -e coverage" - displayName: create coverag report - condition: succeededOrFailed() - - - ${{ if startsWith(parameters.tox, 'py') }}: - - - task: CopyFiles@2 - displayName: move coverage files into staging area - condition: succeededOrFailed() - inputs: - sourceFolder: $(System.DefaultWorkingDirectory)/.tox - contents: | - .coverage - coverage.xml - targetFolder: $(Build.StagingDirectory) - - - task: PublishBuildArtifacts@1 - displayName: publish coverage file - condition: succeededOrFailed() - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - ArtifactName: ${{ format('coverage-{0}-{1}', parameters.os, parameters.tox) }} diff --git a/tox.ini b/tox.ini index 73c4af212..aa1ea50da 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ extras = testing commands = pytest \ --cov "{envsitepackagesdir}/tox" \ --cov-config "{toxinidir}/tox.ini" \ - --junitxml {env:JUNIT_XML_FILE:{toxworkdir}/.test.{envname}.xml} \ + --junitxml {toxworkdir}/junit.{envname}.xml \ -n={env:PYTEST_XDIST_PROC_NR:auto} \ {posargs:.} From 4b29f2dad056368a3512069cb44c0a5957013fba Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Wed, 1 May 2019 06:30:31 -0400 Subject: [PATCH 15/72] fix compound opts handling with pass-through options in provision --- src/tox/config/__init__.py | 8 ++++---- src/tox/session/commands/show_env.py | 7 ++++--- tests/unit/session/test_parallel.py | 2 ++ tests/unit/session/test_session.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 574dc9526..919aea192 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -117,10 +117,10 @@ def add_testenv_attribute_obj(self, obj): self._testenv_attr.append(obj) def parse_cli(self, args, strict=False): - if strict or WITHIN_PROVISION: - return self.argparser.parse_args(args) - else: - return self.argparser.parse_known_args(args)[0] + args, argv = self.argparser.parse_known_args(args) + if argv and (strict or WITHIN_PROVISION): + self.argparser.error("unrecognized arguments: %s".format(" ".join(argv))) + return args def _format_help(self): return self.argparser.format_help() diff --git a/src/tox/session/commands/show_env.py b/src/tox/session/commands/show_env.py index 72e49db3e..ae05c84db 100644 --- a/src/tox/session/commands/show_env.py +++ b/src/tox/session/commands/show_env.py @@ -9,9 +9,9 @@ def show_envs(config, all_envs=False, description=False): ignore = {config.isolated_build_env, config.provision_tox_env}.union(default) extra = [e for e in env_conf if e not in ignore] if all_envs else [] - if description: + if description and default: report.line("default environments:") - max_length = max(len(env) for env in (default + extra)) + max_length = max(len(env) for env in (default + extra)) def report_env(e): if description: @@ -25,7 +25,8 @@ def report_env(e): report_env(e) if all_envs and extra: if description: - report.line("") + if default: + report.line("") report.line("additional environments:") for e in extra: report_env(e) diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index 58435333c..e5281ce98 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -3,6 +3,7 @@ import sys import pytest +from flaky import flaky def test_parallel(cmd, initproj): @@ -160,6 +161,7 @@ def test_parallel_recreate(cmd, initproj, monkeypatch): assert not ({f.basename for f in after} - {f.basename for f in end}) +@flaky(max_runs=3) def test_parallel_show_output(cmd, initproj, monkeypatch): monkeypatch.setenv(str("_TOX_SKIP_ENV_CREATION_TEST"), str("1")) tox_ini = """\ diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index 4614c18ab..b90430395 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -365,6 +365,6 @@ def test_help_compound_ve_works(cmd, initproj, monkeypatch): assert not result.err assert result.outlines[0].startswith("using") assert result.outlines[1].startswith("using") - assert result.outlines[2] == "default environments:" + assert result.outlines[2] == "additional environments:" assert result.outlines[3] == "py -> [no description]" assert len(result.outlines) == 4 From e5b523c77b133efcadbccbb2e282774f8834a1e3 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sat, 4 May 2019 08:32:53 -0700 Subject: [PATCH 16/72] Update Black URLs (#1288) > Black, your uncompromising #Python code formatter, was a project > created with the community in mind from Day 1. Today we moved it under > the PSF umbrella. It's now available on GitHub under > https://github.com/python/black/ . You install and use it just like > before. https://twitter.com/llanga/status/1123980466292445190 --- .pre-commit-config.yaml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df0a4ed2b..189d79cb1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: -- repo: https://github.com/ambv/black +- repo: https://github.com/python/black rev: 19.3b0 hooks: - id: black diff --git a/README.md b/README.md index 7e643fa92..224f713e0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ status](https://dev.azure.com/toxdev/tox/_apis/build/status/tox%20ci?branchName= [![Documentation status](https://readthedocs.org/projects/tox/badge/?version=latest&style=flat-square)](https://tox.readthedocs.io/en/latest/?badge=latest) [![Code style: -black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) +black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) Date: Fri, 10 May 2019 20:25:37 +0100 Subject: [PATCH 17/72] move to improved Azure template (#1286) * move to improved Azure template * add architecture support, log discovery * fix * fix * fix * debug --- azure-pipelines.yml | 57 +++---- docs/changelog/1290.feature.rst | 18 +++ src/tox/config/__init__.py | 36 ++++- src/tox/constants.py | 1 + src/tox/helper/get_version.py | 2 + src/tox/interpreters.py | 237 ++++++++++++++++++++++------ src/tox/session/__init__.py | 2 - tests/unit/config/test_config.py | 2 +- tests/unit/session/test_parallel.py | 1 + tests/unit/test_interpreters.py | 78 ++++----- tests/unit/test_venv.py | 2 +- tox.ini | 2 +- 12 files changed, 299 insertions(+), 139 deletions(-) create mode 100644 docs/changelog/1290.feature.rst diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 279df065f..494aa8b6a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,5 +1,4 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) - resources: repositories: - repository: tox @@ -26,47 +25,29 @@ variables: jobs: - template: run-tox-env.yml@tox parameters: + tox_version: '' jobs: - check: - py: '3.7' - toxenvs: - - fix_lint - - docs - - package_description - windows: - coverage: 'coverage' - toxenvs: - - py37 - - py36 - - py35 - - py34 - - py27 - linux: - coverage: 'coverage' - toxenvs: - - py38 - - py37 - - py36 - - py35 - - py34 - - py27 - macOs: - coverage: 'coverage' - toxenvs: - - py37 - - py27 -- template: merge-coverage.yml@tox - parameters: - dependsOn: - - windows - - linux - - macOs + fix_lint: null + docs: null + py37: + image: [linux, windows, macOs] + py27: + image: [linux, windows, macOs] + py36: + image: [linux, windows, macOs] + py35: + image: [linux, windows, macOs] + py34: + image: [linux, windows, macOs] + dev: null + package_description: null + coverage: + with_toxenv: 'coverage' # generate .tox/.coverage, .tox/coverage.xml after test run + for_envs: [py37, py36, py35, py34, py27] - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}: - template: publish-pypi.yml@tox parameters: - external_feed: 'toxdev' - pypi_remote: 'pypi-toxdev' - - dependsOn: - - check - - report_coverage + - dependsOn: [fix_lint, docs, package_description, dev, report_coverage] diff --git a/docs/changelog/1290.feature.rst b/docs/changelog/1290.feature.rst new file mode 100644 index 000000000..028ead6e8 --- /dev/null +++ b/docs/changelog/1290.feature.rst @@ -0,0 +1,18 @@ +Improve python discovery and add architecture support: + - UNIX: + + - First, check if the tox host Python matches. + - Second, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. + - Third, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. + + - Windows: + + - First, check if the tox host Python matches. + - Second, use the ``py.exe`` to list registered interpreters and any of those match. + - Third, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. + - Fourth, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. + - Finally, check for known locations (``c:\python{major}{minor}\python.exe``). + + +tox environment configuration generation is now done in parallel (to alleviate the slowdown due to extra +checks). diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 919aea192..418cca9fd 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -8,10 +8,12 @@ import shlex import string import sys +import traceback import warnings from collections import OrderedDict from fnmatch import fnmatchcase from subprocess import list2cmdline +from threading import Thread import pkg_resources import pluggy @@ -28,6 +30,7 @@ using, verbosity1, ) +from tox.util.path import ensure_empty_dir from .parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY from .parallel import add_parallel_config, add_parallel_flags @@ -1036,6 +1039,9 @@ def line_of_default_to_zero(section, name=None): config.sdistsrc = reader.getpath("sdistsrc", None) config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") + within_parallel = PARALLEL_ENV_VAR_KEY in os.environ + if not within_parallel: + ensure_empty_dir(config.logdir) # determine indexserver dictionary config.indexserver = {"default": IndexServerConfig("default")} @@ -1084,6 +1090,18 @@ def line_of_default_to_zero(section, name=None): known_factors.update(env.split("-")) # configure testenvs + to_do = [] + failures = OrderedDict() + results = {} + cur_self = self + + def run(name, section, subs, config): + try: + results[name] = cur_self.make_envconfig(name, section, subs, config) + except Exception as exception: + failures[name] = (exception, traceback.format_exc()) + + order = [] for name in all_envs: section = "{}{}".format(testenvprefix, name) factors = set(name.split("-")) @@ -1094,7 +1112,23 @@ def line_of_default_to_zero(section, name=None): tox.PYTHON.PY_FACTORS_RE.match(factor) for factor in factors - known_factors ) ): - config.envconfigs[name] = self.make_envconfig(name, section, reader._subs, config) + order.append(name) + thread = Thread(target=run, args=(name, section, reader._subs, config)) + thread.daemon = True + thread.start() + to_do.append(thread) + for thread in to_do: + while thread.is_alive(): + thread.join(timeout=20) + if failures: + raise tox.exception.ConfigError( + "\n".join( + "{} failed with {} at {}".format(key, exc, trace) + for key, (exc, trace) in failures.items() + ) + ) + for name in order: + config.envconfigs[name] = results[name] all_develop = all( name in config.envconfigs and config.envconfigs[name].usedevelop for name in config.envlist diff --git a/src/tox/constants.py b/src/tox/constants.py index ac72058d9..a2c2ed157 100644 --- a/src/tox/constants.py +++ b/src/tox/constants.py @@ -46,6 +46,7 @@ class INFO: DEFAULT_CONFIG_NAME = "tox.ini" CONFIG_CANDIDATES = ("pyproject.toml", "tox.ini", "setup.cfg") IS_WIN = sys.platform == "win32" + IS_PYPY = hasattr(sys, "pypy_version_info") class PIP: diff --git a/src/tox/helper/get_version.py b/src/tox/helper/get_version.py index ef37a796f..3fcc37e43 100644 --- a/src/tox/helper/get_version.py +++ b/src/tox/helper/get_version.py @@ -5,8 +5,10 @@ info = { "executable": sys.executable, + "name": "pypy" if hasattr(sys, "pypy_version_info") else "python", "version_info": list(sys.version_info), "version": sys.version, + "is_64": sys.maxsize > 2 ** 32, "sysplatform": sys.platform, } info_as_dump = json.dumps(info) diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py index ef7c52384..23be42d65 100644 --- a/src/tox/interpreters.py +++ b/src/tox/interpreters.py @@ -1,10 +1,12 @@ from __future__ import unicode_literals -import distutils.util import json +import os import re import subprocess import sys +from collections import defaultdict +from threading import Lock import py @@ -29,6 +31,7 @@ def get_executable(self, envconfig): return self.name2executable[envconfig.envname] except KeyError: exe = self.hook.tox_get_python_executable(envconfig=envconfig) + reporter.verbosity2("{} uses {}".format(envconfig.envname, exe)) self.name2executable[envconfig.envname] = exe return exe @@ -51,7 +54,7 @@ def get_sitepackagesdir(self, info, envdir): try: res = exec_on_interpreter(str(info.executable), SITE_PACKAGE_QUERY_SCRIPT, str(envdir)) except ExecFailed as e: - print("execution failed: {} -- {}".format(e.out, e.err)) + reporter.verbosity1("execution failed: {} -- {}".format(e.out, e.err)) return "" else: return res["dir"] @@ -62,6 +65,7 @@ def run_and_get_interpreter_info(name, executable): try: result = exec_on_interpreter(str(executable), VERSION_QUERY_SCRIPT) result["version_info"] = tuple(result["version_info"]) # fix json dump transformation + del result["name"] del result["version"] except ExecFailed as e: return NoInterpreterInfo(name, executable=e.executable, out=e.out, err=e.err) @@ -94,12 +98,12 @@ def __init__(self, executable, source, out, err): class InterpreterInfo: - def __init__(self, name, executable, version_info, sysplatform): - assert executable and version_info + def __init__(self, name, executable, version_info, sysplatform, is_64): self.name = name self.executable = executable self.version_info = version_info self.sysplatform = sysplatform + self.is_64 = is_64 def __str__(self): return "".format(self.executable, self.version_info) @@ -120,61 +124,204 @@ def __str__(self): return "".format(self.name) +class PythonSpec(object): + def __init__(self, name, major, minor, architecture, path): + self.name = name + self.major = major + self.minor = minor + self.architecture = architecture + self.path = path + + def __repr__(self): + msg = "PythonSpec(name={}, major={}, minor={}, architecture={}, path={})" + return msg.format(self.name, self.major, self.minor, self.architecture, self.path) + + def satisfies(self, req): + if req.is_abs and self.is_abs and self.path != req.path: + return False + if req.name is not None and req.name != self.name: + return False + if req.architecture is not None and req.architecture != self.architecture: + return False + if req.major is not None and req.major != self.major: + return False + if req.minor is not None and req.minor != self.minor: + return False + if req.major is None and req.minor is not None: + return False + return True + + @property + def is_abs(self): + return self.path is not None and os.path.isabs(self.path) + + @classmethod + def from_name(cls, base_python): + name, major, minor, architecture, path = None, None, None, None, None + if os.path.isabs(base_python): + path = base_python + else: + match = re.match(r"(python|pypy|jython)(\d)?(?:\.(\d))?(-(32|64))?", base_python) + if match: + groups = match.groups() + name = groups[0] + major = int(groups[1]) if len(groups) >= 2 and groups[1] is not None else None + minor = int(groups[2]) if len(groups) >= 3 and groups[2] is not None else None + architecture = ( + int(groups[3]) if len(groups) >= 4 and groups[3] is not None else None + ) + else: + path = base_python + return cls(name, major, minor, architecture, path) + + +CURRENT = PythonSpec( + "pypy" if tox.constants.INFO.IS_PYPY else "python", + sys.version_info[0], + sys.version_info[1], + 64 if sys.maxsize > 2 ** 32 else 32, + sys.executable, +) + if not tox.INFO.IS_WIN: @tox.hookimpl def tox_get_python_executable(envconfig): - if envconfig.basepython == "python{}.{}".format(*sys.version_info[0:2]): - return sys.executable - return py.path.local.sysfind(envconfig.basepython) + base_python = envconfig.basepython + spec = PythonSpec.from_name(base_python) + # first, check current + if spec.name is not None and CURRENT.satisfies(spec): + return CURRENT.path + # second check if the literal base python + candidates = [base_python] + # third check if the un-versioned name is good + if spec.name is not None and spec.name != base_python: + candidates.append(spec.name) + return check_with_path(candidates, spec) else: @tox.hookimpl def tox_get_python_executable(envconfig): - if envconfig.basepython == "python{}.{}".format(*sys.version_info[0:2]): - return sys.executable - p = py.path.local.sysfind(envconfig.basepython) - if p: - return p - - # Is this a standard PythonX.Y name? - m = re.match(r"python(\d)(?:\.(\d))?", envconfig.basepython) - groups = [g for g in m.groups() if g] if m else [] - if m: - # The standard names are in predictable places. - actual = r"c:\python{}\python.exe".format("".join(groups)) - else: + base_python = envconfig.basepython + spec = PythonSpec.from_name(base_python) + # first, check current + if spec.name is not None and CURRENT.satisfies(spec): + return CURRENT.path + + # second check if the py.exe has it (only for non path specs) + if spec.path is None: + py_exe = locate_via_py(spec) + if py_exe is not None: + return py_exe + + # third check if the literal base python is on PATH + candidates = [envconfig.basepython] + # fourth check if the name is on PATH + if spec.name is not None and spec.name != base_python: + candidates.append(spec.name) + # or check known locations + if spec.major is not None and spec.minor is not None: + if spec.name == "python": + # The standard names are in predictable places. + candidates.append(r"c:\python{}{}\python.exe".format(spec.major, spec.minor)) + return check_with_path(candidates, spec) + + _PY_AVAILABLE = [] + _PY_LOCK = Lock() - actual = win32map.get(envconfig.basepython, None) - if actual: - actual = py.path.local(actual) - if actual.check(): - return actual - # Use py.exe to determine location - PEP-514 & PEP-397 - if m: - return locate_via_py(*groups) - - # Exceptions to the usual windows mapping - win32map = {"python": sys.executable, "jython": r"c:\jython2.5.1\jython.bat"} - - def locate_via_py(*parts): - ver = "-{}".format(".".join(parts)) - py_exe = distutils.spawn.find_executable("py") + def locate_via_py(spec): + with _PY_LOCK: + if not _PY_AVAILABLE: + _call_py() + _PY_AVAILABLE.append(CURRENT) + for cur_spec in _PY_AVAILABLE: + if cur_spec.satisfies(spec): + return cur_spec.path + + def _call_py(): + py_exe = py.path.local.sysfind("py") if py_exe: - cmd = py_exe, ver, VERSION_QUERY_SCRIPT + cmd = [str(py_exe), "-0p"] proc = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True + cmd, universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) out, err = proc.communicate() if not proc.returncode: - try: - result = json.loads(out) - except ValueError as exception: - failure = exception - else: - return result["executable"] + elements = [ + tuple(j.strip() for j in i.split("\t")) for i in out.splitlines() if i.strip() + ] + if elements: + for ver_arch, exe in elements: + _, version, arch = ver_arch.split("-") + major, minor = version.split(".") + _PY_AVAILABLE.append( + PythonSpec("python", int(major), int(minor), int(arch), exe) + ) + else: + reporter.verbosity1( + "failed {}, error {},\noutput\n:{}\nstderr:\n{}".format( + cmd, proc.returncode, out, err + ) + ) + + +def check_with_path(candidates, spec): + for path in candidates: + base = path + if not os.path.isabs(path): + path = py.path.local.sysfind(path) + reporter.verbosity2(("path found", path)) + if path is not None: + if os.path.exists(str(path)): + cur_spec = exe_spec(path, base) + if cur_spec is not None and cur_spec.satisfies(spec): + return cur_spec.path + else: + reporter.verbosity2("no such file {}".format(path)) + + +_SPECS = {} +_SPECK_LOCK = defaultdict(Lock) + + +def exe_spec(python_exe, base): + if not isinstance(python_exe, str): + python_exe = str(python_exe) + with _SPECK_LOCK[python_exe]: + if python_exe not in _SPECS: + info = get_python_info([python_exe]) + if info is not None: + found = PythonSpec( + info["name"], + info["version_info"][0], + info["version_info"][1], + 64 if info["is_64"] else 32, + info["executable"], + ) + reporter.verbosity2("{} ({}) is {}".format(base, python_exe, info)) else: - failure = "exit code {}".format(proc.returncode) - reporter.info("{!r} cmd {!r} out {!r} err {!r} ".format(failure, cmd, out, err)) + found = None + _SPECS[python_exe] = found + return _SPECS[python_exe] + + +def get_python_info(cmd): + proc = subprocess.Popen( + cmd + [VERSION_QUERY_SCRIPT], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + out, err = proc.communicate() + if not proc.returncode: + try: + result = json.loads(out) + except ValueError as exception: + failure = exception + else: + return result + else: + failure = "exit code {}".format(proc.returncode) + reporter.verbosity1("{!r} cmd {!r} out {!r} err {!r} ".format(failure, cmd, out, err)) diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py index 2a2079cb5..de885c8f1 100644 --- a/src/tox/session/__init__.py +++ b/src/tox/session/__init__.py @@ -24,7 +24,6 @@ from tox.reporter import update_default_reporter from tox.util import set_os_env_var from tox.util.graph import stable_topological_sort -from tox.util.path import ensure_empty_dir from tox.util.stdlib import suppress_output from tox.venv import VirtualEnv @@ -62,7 +61,6 @@ def main(args): try: config = load_config(args) config.logdir.ensure(dir=1) - ensure_empty_dir(config.logdir) with set_os_env_var("TOX_WORK_DIR", config.toxworkdir): session = build_session(config) exit_code = session.runcommand() diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index d46f58640..c64ec1bea 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -1773,7 +1773,7 @@ def test_recursive_substitution_cycle_fails(self, newconfig): deps= {[testing:pytest]deps} """ - with pytest.raises(ValueError): + with pytest.raises(tox.exception.ConfigError): newconfig([], inisource) def test_single_value_from_other_secton(self, newconfig, tmpdir): diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index e5281ce98..ff3af0604 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -30,6 +30,7 @@ def test_parallel(cmd, initproj): result.assert_success() +@flaky(max_runs=3) def test_parallel_live(cmd, initproj): initproj( "pkg123-0.7", diff --git a/tests/unit/test_interpreters.py b/tests/unit/test_interpreters.py index 3e6fb4bd4..1eba27eb6 100644 --- a/tests/unit/test_interpreters.py +++ b/tests/unit/test_interpreters.py @@ -1,5 +1,3 @@ -import distutils.spawn -import inspect import os import subprocess import sys @@ -8,6 +6,7 @@ import pytest import tox +from tox import reporter from tox._pytestplugin import mark_dont_run_on_posix from tox.config import get_plugin_manager from tox.interpreters import ( @@ -18,6 +17,7 @@ run_and_get_interpreter_info, tox_get_python_executable, ) +from tox.reporter import Verbosity @pytest.fixture(name="interpreters") @@ -28,40 +28,16 @@ def create_interpreters_instance(): @mark_dont_run_on_posix def test_locate_via_py(monkeypatch): - from tox.interpreters import locate_via_py + import tox.interpreters - def fake_find_exe(exe): - assert exe == "py" - return "py" + spec = tox.interpreters.CURRENT + del tox.interpreters._PY_AVAILABLE[:] + exe = tox.interpreters.locate_via_py(spec) + assert exe + assert len(tox.interpreters._PY_AVAILABLE) - from tox.helper import get_version - - def fake_popen(cmd, stdout, stderr, universal_newlines): - fake_popen.last_call = cmd[:3] - - # need to pipe all stdout to collect the version information & need to - # do the same for stderr output to avoid it being forwarded as the - # current process's output, e.g. when the python launcher reports the - # requested Python interpreter not being installed on the system - assert stdout is subprocess.PIPE - assert stderr is subprocess.PIPE - assert universal_newlines is True - - class proc: - returncode = 0 - - @staticmethod - def communicate(): - return get_version.info_as_dump, None - - return proc - - monkeypatch.setattr(distutils.spawn, "find_executable", fake_find_exe) - monkeypatch.setattr(subprocess, "Popen", fake_popen) - assert locate_via_py("3", "6") == sys.executable - assert fake_popen.last_call == ("py", "-3.6", inspect.getsourcefile(get_version)) - assert locate_via_py("3") == sys.executable - assert fake_popen.last_call == ("py", "-3", inspect.getsourcefile(get_version)) + monkeypatch.setattr(tox.interpreters, "_call_py", None) + assert tox.interpreters.locate_via_py(spec) def test_tox_get_python_executable(): @@ -97,7 +73,8 @@ def assert_version_in_output(exe, version): for major in (2, 3): name = "python{}".format(major) if tox.INFO.IS_WIN: - if subprocess.call(("py", "-{}".format(major), "-c", "")): + error_code = subprocess.call(("py", "-{}".format(major), "-c", "")) + if error_code: continue elif not py.path.local.sysfind(name): continue @@ -106,19 +83,25 @@ def assert_version_in_output(exe, version): assert_version_in_output(exe, str(major)) -def test_find_executable_extra(monkeypatch): - @staticmethod - def sysfind(_): - return "hello" - - monkeypatch.setattr(py.path.local, "sysfind", sysfind) +@pytest.mark.skipif("sys.platform == 'win32'", reason="symlink execution unreliable on Windows") +def test_find_alias_on_path(monkeypatch, tmp_path): + reporter.update_default_reporter(Verbosity.DEFAULT, Verbosity.DEBUG) + magic = tmp_path / "magic{}".format(os.path.splitext(sys.executable)[1]) + os.symlink(sys.executable, str(magic)) + monkeypatch.setenv( + str("PATH"), + os.pathsep.join(([str(tmp_path)] + os.environ.get(str("PATH"), "").split(os.pathsep))), + ) class envconfig: - basepython = "1lk23j" + basepython = "magic" envname = "pyxx" - t = tox_get_python_executable(envconfig) - assert t == "hello" + detected = py.path.local.sysfind("magic") + assert detected + + t = tox_get_python_executable(envconfig).lower() + assert t == str(magic).lower() def test_run_and_get_interpreter_info(): @@ -181,12 +164,7 @@ def info( version_info="my-version-info", sysplatform="my-sys-platform", ): - return InterpreterInfo(name, executable, version_info, sysplatform) - - @pytest.mark.parametrize("missing_arg", ("executable", "version_info")) - def test_assert_on_missing_args(self, missing_arg): - with pytest.raises(AssertionError): - self.info(**{missing_arg: None}) + return InterpreterInfo(name, executable, version_info, sysplatform, True) def test_data(self): x = self.info("larry", "moe", "shemp", "curly") diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 18dafb889..05231121f 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -42,7 +42,7 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): venv.getsupportedinterpreter() monkeypatch.undo() monkeypatch.setattr(venv.envconfig, "envname", "py1") - monkeypatch.setattr(venv.envconfig, "basepython", "notexistingpython") + monkeypatch.setattr(venv.envconfig, "basepython", "notexisting") with pytest.raises(tox.exception.InterpreterNotFound): venv.getsupportedinterpreter() monkeypatch.undo() diff --git a/tox.ini b/tox.ini index aa1ea50da..02cab4c37 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ setenv = PIP_DISABLE_VERSION_CHECK = 1 COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} VIRTUALENV_NO_DOWNLOAD = 1 passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_* -deps = +deps = pip == 19.0.3 extras = testing commands = pytest \ --cov "{envsitepackagesdir}/tox" \ From 185d1e3bfbbe46a479c1abec1e236b782e45377c Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 13 May 2019 11:32:46 +0100 Subject: [PATCH 18/72] release 3.10.0 --- azure-pipelines.yml | 6 ++--- docs/changelog.rst | 41 +++++++++++++++++++++++++++++++++ docs/changelog/1270.bugfix.rst | 2 -- docs/changelog/1290.feature.rst | 18 --------------- docs/changelog/448.feature.rst | 3 --- docs/changelog/720.bugfix.rst | 1 - 6 files changed, 44 insertions(+), 27 deletions(-) delete mode 100644 docs/changelog/1270.bugfix.rst delete mode 100644 docs/changelog/1290.feature.rst delete mode 100644 docs/changelog/448.feature.rst delete mode 100644 docs/changelog/720.bugfix.rst diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 494aa8b6a..bce5e39b6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,6 +48,6 @@ jobs: - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags/') }}: - template: publish-pypi.yml@tox parameters: - - external_feed: 'toxdev' - - pypi_remote: 'pypi-toxdev' - - dependsOn: [fix_lint, docs, package_description, dev, report_coverage] + external_feed: 'toxdev' + pypi_remote: 'pypi-toxdev' + dependsOn: [fix_lint, docs, package_description, dev, report_coverage] diff --git a/docs/changelog.rst b/docs/changelog.rst index f45536d6d..2263d0d83 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,47 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.10.0 (2019-05-13) +-------------------- + +Bugfixes +^^^^^^^^ + +- fix for ``tox -l`` command: do not allow setting the ``TOXENV`` or the ``-e`` flag to override the listed default environment variables, they still show up under extra if non defined target - by :user:`gaborbernat` + `#720 `_ +- tox ignores unknown CLI arguments when provisioning is on and outside of the provisioned environment (allowing + provisioning arguments to be forwarded freely) - by :user:`gaborbernat` + `#1270 `_ + + +Features +^^^^^^^^ + +- Virtual environments created now no longer upgrade pip/wheel/setuptools to the latest version. Instead the start + packages after virtualenv creation now is whatever virtualenv has bundled in. This allows faster virtualenv + creation and builds that are easier to reproduce. + `#448 `_ +- Improve python discovery and add architecture support: + - UNIX: + + - First, check if the tox host Python matches. + - Second, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. + - Third, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. + + - Windows: + + - First, check if the tox host Python matches. + - Second, use the ``py.exe`` to list registered interpreters and any of those match. + - Third, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. + - Fourth, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. + - Finally, check for known locations (``c:\python{major}{minor}\python.exe``). + + + tox environment configuration generation is now done in parallel (to alleviate the slowdown due to extra + checks). + `#1290 `_ + + v3.9.0 (2019-04-17) ------------------- diff --git a/docs/changelog/1270.bugfix.rst b/docs/changelog/1270.bugfix.rst deleted file mode 100644 index 38afe0661..000000000 --- a/docs/changelog/1270.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -tox ignores unknown CLI arguments when provisioning is on and outside of the provisioned environment (allowing -provisioning arguments to be forwarded freely) - by :user:`gaborbernat` diff --git a/docs/changelog/1290.feature.rst b/docs/changelog/1290.feature.rst deleted file mode 100644 index 028ead6e8..000000000 --- a/docs/changelog/1290.feature.rst +++ /dev/null @@ -1,18 +0,0 @@ -Improve python discovery and add architecture support: - - UNIX: - - - First, check if the tox host Python matches. - - Second, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. - - Third, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. - - - Windows: - - - First, check if the tox host Python matches. - - Second, use the ``py.exe`` to list registered interpreters and any of those match. - - Third, check if the the canonical name (e.g. ``python3.7``, ``python3``) matches or the base python is an absolute path, use that. - - Fourth, check if the the canonical name without version matches (e.g. ``python``, ``pypy``) matches. - - Finally, check for known locations (``c:\python{major}{minor}\python.exe``). - - -tox environment configuration generation is now done in parallel (to alleviate the slowdown due to extra -checks). diff --git a/docs/changelog/448.feature.rst b/docs/changelog/448.feature.rst deleted file mode 100644 index 02d87e13c..000000000 --- a/docs/changelog/448.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -Virtual environments created now no longer upgrade pip/wheel/setuptools to the latest version. Instead the start -packages after virtualenv creation now is whatever virtualenv has bundled in. This allows faster virtualenv -creation and builds that are easier to reproduce. diff --git a/docs/changelog/720.bugfix.rst b/docs/changelog/720.bugfix.rst deleted file mode 100644 index 6b17d8a70..000000000 --- a/docs/changelog/720.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -fix for ``tox -l`` command: do not allow setting the ``TOXENV`` or the ``-e`` flag to override the listed default environment variables, they still show up under extra if non defined target - by :user:`gaborbernat` From dd6619346306181c7d8313a86afa4ad7be6f2178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Wed, 15 May 2019 17:30:50 +0100 Subject: [PATCH 19/72] show config now shows all config, filter-able, contains host tox python and package versions (#1298) and package versions --- docs/changelog/1298.feature.rst | 11 +++ src/tox/config/__init__.py | 26 +++--- src/tox/session/commands/show_config.py | 96 ++++++++++++++----- tests/unit/config/test_config.py | 49 ---------- tests/unit/session/test_show_config.py | 117 ++++++++++++++++++++++++ tox.ini | 2 +- 6 files changed, 215 insertions(+), 86 deletions(-) create mode 100644 docs/changelog/1298.feature.rst create mode 100644 tests/unit/session/test_show_config.py diff --git a/docs/changelog/1298.feature.rst b/docs/changelog/1298.feature.rst new file mode 100644 index 000000000..1375c507c --- /dev/null +++ b/docs/changelog/1298.feature.rst @@ -0,0 +1,11 @@ +``--showconfig`` overhaul: + +- now fully generated via the config parser, so anyone can load it by using the built-in python config parser +- the ``tox`` section contains all configuration data from config +- the ``tox`` section contains a ``host_python`` key detailing the path of the host python +- the ``tox:version`` section contains the versions of all packages tox depends on with their version +- passing ``-l`` now allows only listing default target envs +- allows showing config for a given set of tox environments only via the ``-e`` cli flag or the ``TOXENV`` environment + variable, in this case the ``tox`` and ``tox:version`` section is only shown if at least one verbosity flag is passed + +this should help inspecting the options. diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 418cca9fd..6551e0cd1 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -392,7 +392,8 @@ def tox_addoption(parser): parser.add_argument( "--showconfig", action="store_true", - help="show configuration information for all environments. ", + help="show live configuration (by default all env, with -l only default targets," + " specific via TOXENV/-e)", ) parser.add_argument( "-l", @@ -1077,7 +1078,8 @@ def line_of_default_to_zero(section, name=None): self.handle_provision(config, reader) self.parse_build_isolation(config, reader) - config.envlist, all_envs, config.envlist_default = self._getenvdata(reader, config) + res = self._getenvdata(reader, config) + config.envlist, all_envs, config.envlist_default, config.envlist_explicit = res # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -1268,18 +1270,19 @@ def _getenvdata(self, reader, config): from_config = reader.getstring("envlist", replace=False) env_list = [] + envlist_explicit = False if (from_option and "ALL" in from_option) or ( not from_option and from_environ and "ALL" in from_environ.split(",") ): all_envs = self._getallenvs(reader) else: candidates = ( - os.environ.get(PARALLEL_ENV_VAR_KEY), - from_option, - from_environ, - from_config, + (os.environ.get(PARALLEL_ENV_VAR_KEY), True), + (from_option, True), + (from_environ, True), + (from_config, False), ) - env_str = next((i for i in candidates if i), []) + env_str, envlist_explicit = next(((i, e) for i, e in candidates if i), ([], False)) env_list = _split_env(env_str) all_envs = self._getallenvs(reader, env_list) @@ -1293,7 +1296,7 @@ def _getenvdata(self, reader, config): if config.isolated_build is True and package_env in env_list: msg = "isolated_build_env {} cannot be part of envlist".format(package_env) raise tox.exception.ConfigError(msg) - return env_list, all_envs, _split_env(from_config) + return env_list, all_envs, _split_env(from_config), envlist_explicit def _split_env(env): @@ -1353,21 +1356,22 @@ def __init__(self, name, indexserver=None): self.name = name self.indexserver = indexserver - def __str__(self): + def __repr__(self): if self.indexserver: if self.indexserver.name == "default": return self.name return ":{}:{}".format(self.indexserver.name, self.name) return str(self.name) - __repr__ = __str__ - class IndexServerConfig: def __init__(self, name, url=None): self.name = name self.url = url + def __repr__(self): + return "IndexServerConfig(name={}, url={})".format(self.name, self.url) + is_section_substitution = re.compile(r"{\[[^{}\s]+\]\S+?}").match """Check value matches substitution form of referencing value from other section. diff --git a/src/tox/session/commands/show_config.py b/src/tox/session/commands/show_config.py index d588ac80d..f307cc186 100644 --- a/src/tox/session/commands/show_config.py +++ b/src/tox/session/commands/show_config.py @@ -1,31 +1,77 @@ -import subprocess import sys +from collections import OrderedDict -from tox import reporter as report -from tox.version import __version__ +from six import StringIO +from six.moves import configparser + +from tox import reporter + +DO_NOT_SHOW_CONFIG_ATTRIBUTES = ( + "interpreters", + "envconfigs", + "envlist", + "pluginmanager", + "envlist_explicit", +) def show_config(config): - info_versions() - report.keyvalue("config-file:", config.option.configfile) - report.keyvalue("toxinipath: ", config.toxinipath) - report.keyvalue("toxinidir: ", config.toxinidir) - report.keyvalue("toxworkdir: ", config.toxworkdir) - report.keyvalue("setupdir: ", config.setupdir) - report.keyvalue("distshare: ", config.distshare) - report.keyvalue("skipsdist: ", config.skipsdist) - report.line("") - for envconfig in config.envconfigs.values(): - report.line("[testenv:{}]".format(envconfig.envname), bold=True) - for attr in config._parser._testenv_attr: - report.line(" {:<15} = {}".format(attr.name, getattr(envconfig, attr.name))) - - -def info_versions(): - versions = ["tox-{}".format(__version__)] - proc = subprocess.Popen( - (sys.executable, "-m", "virtualenv", "--version"), stdout=subprocess.PIPE + parser = configparser.ConfigParser() + + if not config.envlist_explicit or reporter.verbosity() >= reporter.Verbosity.INFO: + tox_info(config, parser) + version_info(parser) + tox_envs_info(config, parser) + + content = StringIO() + parser.write(content) + value = content.getvalue().rstrip() + reporter.verbosity0(value) + + +def tox_envs_info(config, parser): + if config.envlist_explicit: + env_list = config.envlist + elif config.option.listenvs: + env_list = config.envlist_default + else: + env_list = list(config.envconfigs.keys()) + for name in env_list: + env_config = config.envconfigs[name] + values = OrderedDict( + (attr.name, str(getattr(env_config, attr.name))) + for attr in config._parser._testenv_attr + ) + section = "testenv:{}".format(name) + set_section(parser, section, values) + + +def tox_info(config, parser): + info = OrderedDict( + (i, str(getattr(config, i))) + for i in sorted(dir(config)) + if not i.startswith("_") and i not in DO_NOT_SHOW_CONFIG_ATTRIBUTES ) - out, _ = proc.communicate() - versions.append("virtualenv-{}".format(out.decode("UTF-8").strip())) - report.keyvalue("tool-versions:", " ".join(versions)) + info["host_python"] = sys.executable + set_section(parser, "tox", info) + + +def version_info(parser): + import pkg_resources + + versions = OrderedDict() + visited = set() + to_visit = {"tox"} + while to_visit: + current = to_visit.pop() + visited.add(current) + current_dist = pkg_resources.get_distribution(current) + to_visit.update(i.name for i in current_dist.requires() if i.name not in visited) + versions[current] = current_dist.version + set_section(parser, "tox:versions", versions) + + +def set_section(parser, section, values): + parser.add_section(section) + for key, value in values.items(): + parser.set(section, key, value) diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index c64ec1bea..206fde8df 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -2722,12 +2722,6 @@ class MockEggInfo: assert "some-repr" in version_info assert "1.0" in version_info - def test_config_specific_ini(self, tmpdir, cmd): - ini = tmpdir.ensure("hello.ini") - result = cmd("-c", ini, "--showconfig") - assert not result.ret - assert result.outlines[1] == "config-file: {}".format(ini) - def test_no_tox_ini(self, cmd, initproj): initproj("noini-0.5") result = cmd() @@ -2736,49 +2730,6 @@ def test_no_tox_ini(self, cmd, initproj): assert result.err == msg assert not result.out - def test_override_workdir(self, cmd, initproj): - baddir = "badworkdir-123" - gooddir = "overridden-234" - initproj( - "overrideworkdir-0.5", - filedefs={ - "tox.ini": """ - [tox] - toxworkdir={} - """.format( - baddir - ) - }, - ) - result = cmd("--workdir", gooddir, "--showconfig") - assert not result.ret - assert gooddir in result.out - assert baddir not in result.out - assert py.path.local(gooddir).check() - assert not py.path.local(baddir).check() - - def test_showconfig_with_force_dep_version(self, cmd, initproj): - initproj( - "force_dep_version", - filedefs={ - "tox.ini": """ - [tox] - - [testenv] - deps= - dep1==2.3 - dep2 - """ - }, - ) - result = cmd("--showconfig") - result.assert_success(is_run_test_env=False) - assert any(re.match(r".*deps.*dep1==2.3, dep2.*", l) for l in result.outlines) - # override dep1 specific version, and force version for dep2 - result = cmd("--showconfig", "--force-dep=dep1", "--force-dep=dep2==5.0") - result.assert_success(is_run_test_env=False) - assert any(re.match(r".*deps.*dep1, dep2==5.0.*", l) for l in result.outlines) - @pytest.mark.parametrize( "cli_args,run_envlist", diff --git a/tests/unit/session/test_show_config.py b/tests/unit/session/test_show_config.py new file mode 100644 index 000000000..1a01d1d89 --- /dev/null +++ b/tests/unit/session/test_show_config.py @@ -0,0 +1,117 @@ +import py +import pytest +from six import StringIO +from six.moves import configparser + + +def load_config(args, cmd): + result = cmd(*args) + result.assert_success(is_run_test_env=False) + parser = configparser.ConfigParser() + output = StringIO(result.out) + parser.readfp(output) + return parser + + +def test_showconfig_with_force_dep_version(cmd, initproj): + initproj( + "force_dep_version", + filedefs={ + "tox.ini": """ + [tox] + + [testenv] + deps= + dep1==2.3 + dep2 + """ + }, + ) + parser = load_config(("--showconfig",), cmd) + assert parser.get("testenv:python", "deps") == "[dep1==2.3, dep2]" + + parser = load_config(("--showconfig", "--force-dep=dep1", "--force-dep=dep2==5.0"), cmd) + assert parser.get("testenv:python", "deps") == "[dep1, dep2==5.0]" + + +@pytest.fixture() +def setup_mixed_conf(initproj): + initproj( + "force_dep_version", + filedefs={ + "tox.ini": """ + [tox] + envlist = py37,py27,pypi,docs + + [testenv:notincluded] + changedir = whatever + + [testenv:docs] + changedir = docs + """ + }, + ) + + +@pytest.mark.parametrize( + "args, expected", + [ + ( + ["--showconfig"], + [ + "tox", + "tox:versions", + "testenv:py37", + "testenv:py27", + "testenv:pypi", + "testenv:docs", + "testenv:notincluded", + ], + ), + ( + ["--showconfig", "-l"], + [ + "tox", + "tox:versions", + "testenv:py37", + "testenv:py27", + "testenv:pypi", + "testenv:docs", + ], + ), + (["--showconfig", "-e", "py37,py36"], ["testenv:py37", "testenv:py36"]), + ], + ids=["all", "default_only", "-e"], +) +def test_showconfig(cmd, setup_mixed_conf, args, expected): + parser = load_config(args, cmd) + found_sections = parser.sections() + assert found_sections == expected + + +def test_config_specific_ini(tmpdir, cmd): + ini = tmpdir.ensure("hello.ini") + output = load_config(("-c", ini, "--showconfig"), cmd) + assert output.get("tox", "toxinipath") == ini + + +def test_override_workdir(cmd, initproj): + baddir = "badworkdir-123" + gooddir = "overridden-234" + initproj( + "overrideworkdir-0.5", + filedefs={ + "tox.ini": """ + [tox] + toxworkdir={} + """.format( + baddir + ) + }, + ) + result = cmd("--workdir", gooddir, "--showconfig") + assert not result.ret + assert gooddir in result.out + assert baddir not in result.out + assert py.path.local(gooddir).check() + assert not py.path.local(baddir).check() diff --git a/tox.ini b/tox.ini index 02cab4c37..dc192ae09 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ setenv = PIP_DISABLE_VERSION_CHECK = 1 COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} VIRTUALENV_NO_DOWNLOAD = 1 passenv = http_proxy https_proxy no_proxy SSL_CERT_FILE PYTEST_* -deps = pip == 19.0.3 +deps = pip == 19.1.1 extras = testing commands = pytest \ --cov "{envsitepackagesdir}/tox" \ From 67d7c3d1aca05d45c9d17a96bdf720564c29de57 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Wed, 15 May 2019 17:32:15 +0100 Subject: [PATCH 20/72] release 3.11.0 --- docs/changelog.rst | 20 ++++++++++++++++++++ docs/changelog/1298.feature.rst | 11 ----------- 2 files changed, 20 insertions(+), 11 deletions(-) delete mode 100644 docs/changelog/1298.feature.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 2263d0d83..38f61de14 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,26 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.11.0 (2019-05-15) +-------------------- + +Features +^^^^^^^^ + +- ``--showconfig`` overhaul: + + - now fully generated via the config parser, so anyone can load it by using the built-in python config parser + - the ``tox`` section contains all configuration data from config + - the ``tox`` section contains a ``host_python`` key detailing the path of the host python + - the ``tox:version`` section contains the versions of all packages tox depends on with their version + - passing ``-l`` now allows only listing default target envs + - allows showing config for a given set of tox environments only via the ``-e`` cli flag or the ``TOXENV`` environment + variable, in this case the ``tox`` and ``tox:version`` section is only shown if at least one verbosity flag is passed + + this should help inspecting the options. + `#1298 `_ + + v3.10.0 (2019-05-13) -------------------- diff --git a/docs/changelog/1298.feature.rst b/docs/changelog/1298.feature.rst deleted file mode 100644 index 1375c507c..000000000 --- a/docs/changelog/1298.feature.rst +++ /dev/null @@ -1,11 +0,0 @@ -``--showconfig`` overhaul: - -- now fully generated via the config parser, so anyone can load it by using the built-in python config parser -- the ``tox`` section contains all configuration data from config -- the ``tox`` section contains a ``host_python`` key detailing the path of the host python -- the ``tox:version`` section contains the versions of all packages tox depends on with their version -- passing ``-l`` now allows only listing default target envs -- allows showing config for a given set of tox environments only via the ``-e`` cli flag or the ``TOXENV`` environment - variable, in this case the ``tox`` and ``tox:version`` section is only shown if at least one verbosity flag is passed - -this should help inspecting the options. From a8b34cc5690cef34fa91ca0f1950a1fdde0f8b72 Mon Sep 17 00:00:00 2001 From: Ashley Whetter Date: Thu, 16 May 2019 02:14:49 -0700 Subject: [PATCH 21/72] Fixed result of tox_get_python_executable not being used to make virtualenvs (#1301) * Fixed result of tox_get_python_executable not being used to make virtualenvs * fix formatting and add changelog --- CONTRIBUTORS | 1 + docs/changelog/1301.bugfix.rst | 1 + src/tox/interpreters.py | 1 + src/tox/logs/env.py | 1 + tests/unit/test_interpreters.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 35 insertions(+) create mode 100644 docs/changelog/1301.bugfix.rst diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e3301f9fc..fcd11b187 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -6,6 +6,7 @@ Allan Feldman Andrii Soldatenko Anthon van der Neuth Anthony Sottile +Ashley Whetter Asmund Grammeltwedt Barry Warsaw Bartolome Sanchez Salado diff --git a/docs/changelog/1301.bugfix.rst b/docs/changelog/1301.bugfix.rst new file mode 100644 index 000000000..c666827f6 --- /dev/null +++ b/docs/changelog/1301.bugfix.rst @@ -0,0 +1 @@ +When creating virtual environments we no longer ask the python to tell its path, but rather use the discovered path. diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py index 23be42d65..14f9eb422 100644 --- a/src/tox/interpreters.py +++ b/src/tox/interpreters.py @@ -67,6 +67,7 @@ def run_and_get_interpreter_info(name, executable): result["version_info"] = tuple(result["version_info"]) # fix json dump transformation del result["name"] del result["version"] + result["executable"] = str(executable) except ExecFailed as e: return NoInterpreterInfo(name, executable=e.executable, out=e.out, err=e.err) else: diff --git a/src/tox/logs/env.py b/src/tox/logs/env.py index bbdc0be52..f134440a7 100644 --- a/src/tox/logs/env.py +++ b/src/tox/logs/env.py @@ -20,6 +20,7 @@ def set_python_info(self, python_executable): cmd = [str(python_executable), VERSION_QUERY_SCRIPT] result = subprocess.check_output(cmd, universal_newlines=True) answer = json.loads(result) + answer["executable"] = python_executable self.dict["python"] = answer def get_commandlog(self, name): diff --git a/tests/unit/test_interpreters.py b/tests/unit/test_interpreters.py index 1eba27eb6..e7928cf75 100644 --- a/tests/unit/test_interpreters.py +++ b/tests/unit/test_interpreters.py @@ -1,4 +1,7 @@ +from __future__ import unicode_literals + import os +import stat import subprocess import sys @@ -137,6 +140,34 @@ class envconfig: assert not info.executable assert isinstance(info, NoInterpreterInfo) + @pytest.mark.skipif("sys.platform == 'win32'", reason="Uses a unix only wrapper") + def test_get_info_uses_hook_path(self, tmp_path): + magic = tmp_path / "magic{}".format(os.path.splitext(sys.executable)[1]) + wrapper = ( + "#!{executable}\n" + "import subprocess\n" + "import sys\n" + 'sys.exit(subprocess.call(["{executable}"] + sys.argv[1:]))\n' + ).format(executable=sys.executable) + magic.write_text(wrapper) + magic.chmod(magic.stat().st_mode | stat.S_IEXEC) + + class MockHook: + def tox_get_python_executable(self, envconfig): + return str(magic) + + class envconfig: + basepython = sys.executable + envname = "magicpy" + + # Check that the wrapper is working first. + # If it isn't, the default is to return the passed path anyway. + subprocess.check_call([str(magic), "--help"]) + + interpreters = Interpreters(hook=MockHook()) + info = interpreters.get_info(envconfig) + assert info.executable == str(magic) + def test_get_sitepackagesdir_error(self, interpreters): class envconfig: basepython = sys.executable From af4feaca4c1456f5df2b2ed58ff0196cf901ab86 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 16 May 2019 10:15:44 +0100 Subject: [PATCH 22/72] release 3.11.1 --- docs/changelog.rst | 10 ++++++++++ docs/changelog/1301.bugfix.rst | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 docs/changelog/1301.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 38f61de14..690389ab3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,16 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.11.1 (2019-05-16) +-------------------- + +Bugfixes +^^^^^^^^ + +- When creating virtual environments we no longer ask the python to tell its path, but rather use the discovered path. + `#1301 `_ + + v3.11.0 (2019-05-15) -------------------- diff --git a/docs/changelog/1301.bugfix.rst b/docs/changelog/1301.bugfix.rst deleted file mode 100644 index c666827f6..000000000 --- a/docs/changelog/1301.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -When creating virtual environments we no longer ask the python to tell its path, but rather use the discovered path. From c3da8e766350165f5e2ab717a2e817d2a8c8c193 Mon Sep 17 00:00:00 2001 From: zeroshift Date: Thu, 23 May 2019 11:13:23 +0200 Subject: [PATCH 23/72] Add environment variable to disable parallel spinner for CI tools (#1311) * Adding TOX_PARALLEL_NO_SPINNER to disable spinner for CI tools. * Adding unit tests for TOX_PARALLEL_NO_SPINNER env var. * Updating CONTRIBUTORS and changelog for TOX_PARALLEL_NO_SPINNER env var. --- CONTRIBUTORS | 1 + docs/changelog/1184.feature.rst | 1 + src/tox/session/commands/help.py | 1 + src/tox/session/commands/run/parallel.py | 5 ++++- tests/unit/config/test_config.py | 11 +++++++++++ tests/unit/session/test_parallel.py | 25 ++++++++++++++++++++++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1184.feature.rst diff --git a/CONTRIBUTORS b/CONTRIBUTORS index fcd11b187..a95197ffb 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -60,6 +60,7 @@ Mikhail Kyshtymov Monty Taylor Morgan Fainberg Nick Douma +Nick Prendergast Oliver Bestwalter Paweł Adamczak Philip Thiem diff --git a/docs/changelog/1184.feature.rst b/docs/changelog/1184.feature.rst new file mode 100644 index 000000000..736a7a581 --- /dev/null +++ b/docs/changelog/1184.feature.rst @@ -0,0 +1 @@ +Adding ```TOX_PARALLEL_NO_SPINNER``` environment variable to disable the spinner in parallel mode for the purposes of clean output when using CI tools - by :user:`zeroshift` diff --git a/src/tox/session/commands/help.py b/src/tox/session/commands/help.py index bd9f55848..6043e9f64 100644 --- a/src/tox/session/commands/help.py +++ b/src/tox/session/commands/help.py @@ -11,3 +11,4 @@ def show_help(config): "passed into test command environments" ) reporter.line("PY_COLORS: 0 disable colorized output, 1 enable (default)") + reporter.line("TOX_PARALLEL_NO_SPINNER: 1 disable spinner for CI, 0 enable (default)") diff --git a/src/tox/session/commands/run/parallel.py b/src/tox/session/commands/run/parallel.py index c01eb9dd8..e307c1cfd 100644 --- a/src/tox/session/commands/run/parallel.py +++ b/src/tox/session/commands/run/parallel.py @@ -13,6 +13,7 @@ def run_parallel(config, venv_dict): """here we'll just start parallel sub-processes""" live_out = config.option.parallel_live + disable_spinner = bool(os.environ.get("TOX_PARALLEL_NO_SPINNER") == "1") args = [sys.executable, MAIN_FILE] + config.args try: position = args.index("--") @@ -25,7 +26,9 @@ def run_parallel(config, venv_dict): semaphore = Semaphore(max_parallel) finished = Event() - show_progress = not live_out and reporter.verbosity() > reporter.Verbosity.QUIET + show_progress = ( + not disable_spinner and not live_out and reporter.verbosity() > reporter.Verbosity.QUIET + ) with Spinner(enabled=show_progress) as spinner: diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 206fde8df..09f5bf5c3 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -1164,6 +1164,17 @@ def test_passenv_glob_from_global_env(self, newconfig, monkeypatch): assert "A1" in env.passenv assert "A2" in env.passenv + def test_no_spinner(self, newconfig, monkeypatch): + monkeypatch.setenv("TOX_PARALLEL_NO_SPINNER", "1") + config = newconfig( + """ + [testenv] + passenv = TOX_PARALLEL_NO_SPINNER + """ + ) + env = config.envconfigs["python"] + assert "TOX_PARALLEL_NO_SPINNER" in env.passenv + def test_changedir_override(self, newconfig): config = newconfig( """ diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index ff3af0604..3d8746b50 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -189,3 +189,28 @@ def test_parallel_show_output(cmd, initproj, monkeypatch): assert "stderr env" not in result.out, result.output() assert "stdout always" in result.out, result.output() assert "stderr always" in result.out, result.output() + + +def test_parallel_no_spinner(cmd, initproj, monkeypatch): + monkeypatch.setenv(str("TOX_PARALLEL_NO_SPINNER"), str("1")) + initproj( + "pkg123-0.7", + filedefs={ + "tox.ini": """ + [tox] + envlist = a, b + isolated_build = true + [testenv] + commands=python -c "import sys; print(sys.executable)" + [testenv:b] + depends = a + """, + "pyproject.toml": """ + [build-system] + requires = ["setuptools >= 35.0.2"] + build-backend = 'setuptools.build_meta' + """, + }, + ) + result = cmd("--parallel", "all") + result.assert_success() From 036b03e85f4e5004891ce817798826336fa4d20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 23 May 2019 13:12:57 +0100 Subject: [PATCH 24/72] PEP-514 support via winreg (#1310) * pep-514 implementation * reorganize interpreter * report failures * report failures --- .pre-commit-config.yaml | 12 +- docs/changelog/1306.bugfix.rst | 1 + src/tox/interpreters.py | 328 ------------------ src/tox/interpreters/__init__.py | 125 +++++++ src/tox/interpreters/py_spec.py | 76 ++++ src/tox/interpreters/unix.py | 21 ++ src/tox/interpreters/via_path.py | 71 ++++ src/tox/interpreters/windows/__init__.py | 52 +++ src/tox/interpreters/windows/pep514.py | 165 +++++++++ tests/unit/config/test_config.py | 4 - .../{ => interpreters}/test_interpreters.py | 17 +- .../unit/interpreters/windows/test_pep514.py | 25 ++ .../unit/interpreters/windows/test_windows.py | 20 ++ tests/unit/session/test_show_config.py | 4 +- tox.ini | 7 +- 15 files changed, 570 insertions(+), 358 deletions(-) create mode 100644 docs/changelog/1306.bugfix.rst delete mode 100644 src/tox/interpreters.py create mode 100644 src/tox/interpreters/__init__.py create mode 100644 src/tox/interpreters/py_spec.py create mode 100644 src/tox/interpreters/unix.py create mode 100644 src/tox/interpreters/via_path.py create mode 100644 src/tox/interpreters/windows/__init__.py create mode 100644 src/tox/interpreters/windows/pep514.py rename tests/unit/{ => interpreters}/test_interpreters.py (92%) create mode 100644 tests/unit/interpreters/windows/test_pep514.py create mode 100644 tests/unit/interpreters/windows/test_windows.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 189d79cb1..cc63bdc72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,29 +12,29 @@ repos: additional_dependencies: [black==19.3b0] language_version: python3.7 - repo: https://github.com/asottile/seed-isort-config - rev: v1.7.0 + rev: v1.9.0 hooks: - id: seed-isort-config args: [--application-directories, "src:."] - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.15 + rev: v4.3.20 hooks: - id: isort - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.1.0 + rev: v2.2.3 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: debug-statements - id: flake8 - additional_dependencies: ["flake8-bugbear == 18.8.0"] + additional_dependencies: ["flake8-bugbear == 19.3.0"] language_version: python3.7 - repo: https://github.com/asottile/pyupgrade - rev: v1.12.0 + rev: v1.17.1 hooks: - id: pyupgrade - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.2.0 + rev: v1.4.0 hooks: - id: rst-backticks diff --git a/docs/changelog/1306.bugfix.rst b/docs/changelog/1306.bugfix.rst new file mode 100644 index 000000000..469ca1635 --- /dev/null +++ b/docs/changelog/1306.bugfix.rst @@ -0,0 +1 @@ +Turns out the output of the ``py -0p`` is not stable yet and varies depending on various edge cases. Instead now we read the interpreter values directly from registry via `PEP-514 `_ - by :user:`gaborbernat`. diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py deleted file mode 100644 index 14f9eb422..000000000 --- a/src/tox/interpreters.py +++ /dev/null @@ -1,328 +0,0 @@ -from __future__ import unicode_literals - -import json -import os -import re -import subprocess -import sys -from collections import defaultdict -from threading import Lock - -import py - -import tox -from tox import reporter -from tox.constants import SITE_PACKAGE_QUERY_SCRIPT, VERSION_QUERY_SCRIPT - - -class Interpreters: - def __init__(self, hook): - self.name2executable = {} - self.executable2info = {} - self.hook = hook - - def get_executable(self, envconfig): - """ return path object to the executable for the given - name (e.g. python2.7, python3.6, python etc.) - if name is already an existing path, return name. - If an interpreter cannot be found, return None. - """ - try: - return self.name2executable[envconfig.envname] - except KeyError: - exe = self.hook.tox_get_python_executable(envconfig=envconfig) - reporter.verbosity2("{} uses {}".format(envconfig.envname, exe)) - self.name2executable[envconfig.envname] = exe - return exe - - def get_info(self, envconfig): - executable = self.get_executable(envconfig) - name = envconfig.basepython - if not executable: - return NoInterpreterInfo(name=name) - try: - return self.executable2info[executable] - except KeyError: - info = run_and_get_interpreter_info(name, executable) - self.executable2info[executable] = info - return info - - def get_sitepackagesdir(self, info, envdir): - if not info.executable: - return "" - envdir = str(envdir) - try: - res = exec_on_interpreter(str(info.executable), SITE_PACKAGE_QUERY_SCRIPT, str(envdir)) - except ExecFailed as e: - reporter.verbosity1("execution failed: {} -- {}".format(e.out, e.err)) - return "" - else: - return res["dir"] - - -def run_and_get_interpreter_info(name, executable): - assert executable - try: - result = exec_on_interpreter(str(executable), VERSION_QUERY_SCRIPT) - result["version_info"] = tuple(result["version_info"]) # fix json dump transformation - del result["name"] - del result["version"] - result["executable"] = str(executable) - except ExecFailed as e: - return NoInterpreterInfo(name, executable=e.executable, out=e.out, err=e.err) - else: - return InterpreterInfo(name, **result) - - -def exec_on_interpreter(*args): - from subprocess import Popen, PIPE - - popen = Popen(args, stdout=PIPE, stderr=PIPE, universal_newlines=True) - out, err = popen.communicate() - if popen.returncode: - raise ExecFailed(args[0], args[1:], out, err) - if err: - sys.stderr.write(err) - try: - result = json.loads(out) - except Exception: - raise ExecFailed(args[0], args[1:], out, "could not decode {!r}".format(out)) - return result - - -class ExecFailed(Exception): - def __init__(self, executable, source, out, err): - self.executable = executable - self.source = source - self.out = out - self.err = err - - -class InterpreterInfo: - def __init__(self, name, executable, version_info, sysplatform, is_64): - self.name = name - self.executable = executable - self.version_info = version_info - self.sysplatform = sysplatform - self.is_64 = is_64 - - def __str__(self): - return "".format(self.executable, self.version_info) - - -class NoInterpreterInfo: - def __init__(self, name, executable=None, out=None, err="not found"): - self.name = name - self.executable = executable - self.version_info = None - self.out = out - self.err = err - - def __str__(self): - if self.executable: - return "".format(self.executable) - else: - return "".format(self.name) - - -class PythonSpec(object): - def __init__(self, name, major, minor, architecture, path): - self.name = name - self.major = major - self.minor = minor - self.architecture = architecture - self.path = path - - def __repr__(self): - msg = "PythonSpec(name={}, major={}, minor={}, architecture={}, path={})" - return msg.format(self.name, self.major, self.minor, self.architecture, self.path) - - def satisfies(self, req): - if req.is_abs and self.is_abs and self.path != req.path: - return False - if req.name is not None and req.name != self.name: - return False - if req.architecture is not None and req.architecture != self.architecture: - return False - if req.major is not None and req.major != self.major: - return False - if req.minor is not None and req.minor != self.minor: - return False - if req.major is None and req.minor is not None: - return False - return True - - @property - def is_abs(self): - return self.path is not None and os.path.isabs(self.path) - - @classmethod - def from_name(cls, base_python): - name, major, minor, architecture, path = None, None, None, None, None - if os.path.isabs(base_python): - path = base_python - else: - match = re.match(r"(python|pypy|jython)(\d)?(?:\.(\d))?(-(32|64))?", base_python) - if match: - groups = match.groups() - name = groups[0] - major = int(groups[1]) if len(groups) >= 2 and groups[1] is not None else None - minor = int(groups[2]) if len(groups) >= 3 and groups[2] is not None else None - architecture = ( - int(groups[3]) if len(groups) >= 4 and groups[3] is not None else None - ) - else: - path = base_python - return cls(name, major, minor, architecture, path) - - -CURRENT = PythonSpec( - "pypy" if tox.constants.INFO.IS_PYPY else "python", - sys.version_info[0], - sys.version_info[1], - 64 if sys.maxsize > 2 ** 32 else 32, - sys.executable, -) - -if not tox.INFO.IS_WIN: - - @tox.hookimpl - def tox_get_python_executable(envconfig): - base_python = envconfig.basepython - spec = PythonSpec.from_name(base_python) - # first, check current - if spec.name is not None and CURRENT.satisfies(spec): - return CURRENT.path - # second check if the literal base python - candidates = [base_python] - # third check if the un-versioned name is good - if spec.name is not None and spec.name != base_python: - candidates.append(spec.name) - return check_with_path(candidates, spec) - - -else: - - @tox.hookimpl - def tox_get_python_executable(envconfig): - base_python = envconfig.basepython - spec = PythonSpec.from_name(base_python) - # first, check current - if spec.name is not None and CURRENT.satisfies(spec): - return CURRENT.path - - # second check if the py.exe has it (only for non path specs) - if spec.path is None: - py_exe = locate_via_py(spec) - if py_exe is not None: - return py_exe - - # third check if the literal base python is on PATH - candidates = [envconfig.basepython] - # fourth check if the name is on PATH - if spec.name is not None and spec.name != base_python: - candidates.append(spec.name) - # or check known locations - if spec.major is not None and spec.minor is not None: - if spec.name == "python": - # The standard names are in predictable places. - candidates.append(r"c:\python{}{}\python.exe".format(spec.major, spec.minor)) - return check_with_path(candidates, spec) - - _PY_AVAILABLE = [] - _PY_LOCK = Lock() - - def locate_via_py(spec): - with _PY_LOCK: - if not _PY_AVAILABLE: - _call_py() - _PY_AVAILABLE.append(CURRENT) - for cur_spec in _PY_AVAILABLE: - if cur_spec.satisfies(spec): - return cur_spec.path - - def _call_py(): - py_exe = py.path.local.sysfind("py") - if py_exe: - cmd = [str(py_exe), "-0p"] - proc = subprocess.Popen( - cmd, universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE - ) - out, err = proc.communicate() - if not proc.returncode: - elements = [ - tuple(j.strip() for j in i.split("\t")) for i in out.splitlines() if i.strip() - ] - if elements: - for ver_arch, exe in elements: - _, version, arch = ver_arch.split("-") - major, minor = version.split(".") - _PY_AVAILABLE.append( - PythonSpec("python", int(major), int(minor), int(arch), exe) - ) - else: - reporter.verbosity1( - "failed {}, error {},\noutput\n:{}\nstderr:\n{}".format( - cmd, proc.returncode, out, err - ) - ) - - -def check_with_path(candidates, spec): - for path in candidates: - base = path - if not os.path.isabs(path): - path = py.path.local.sysfind(path) - reporter.verbosity2(("path found", path)) - if path is not None: - if os.path.exists(str(path)): - cur_spec = exe_spec(path, base) - if cur_spec is not None and cur_spec.satisfies(spec): - return cur_spec.path - else: - reporter.verbosity2("no such file {}".format(path)) - - -_SPECS = {} -_SPECK_LOCK = defaultdict(Lock) - - -def exe_spec(python_exe, base): - if not isinstance(python_exe, str): - python_exe = str(python_exe) - with _SPECK_LOCK[python_exe]: - if python_exe not in _SPECS: - info = get_python_info([python_exe]) - if info is not None: - found = PythonSpec( - info["name"], - info["version_info"][0], - info["version_info"][1], - 64 if info["is_64"] else 32, - info["executable"], - ) - reporter.verbosity2("{} ({}) is {}".format(base, python_exe, info)) - else: - found = None - _SPECS[python_exe] = found - return _SPECS[python_exe] - - -def get_python_info(cmd): - proc = subprocess.Popen( - cmd + [VERSION_QUERY_SCRIPT], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - out, err = proc.communicate() - if not proc.returncode: - try: - result = json.loads(out) - except ValueError as exception: - failure = exception - else: - return result - else: - failure = "exit code {}".format(proc.returncode) - reporter.verbosity1("{!r} cmd {!r} out {!r} err {!r} ".format(failure, cmd, out, err)) diff --git a/src/tox/interpreters/__init__.py b/src/tox/interpreters/__init__.py new file mode 100644 index 000000000..b8d12cd59 --- /dev/null +++ b/src/tox/interpreters/__init__.py @@ -0,0 +1,125 @@ +from __future__ import unicode_literals + +import json +import sys + +import tox +from tox import reporter +from tox.constants import SITE_PACKAGE_QUERY_SCRIPT, VERSION_QUERY_SCRIPT + + +class Interpreters: + def __init__(self, hook): + self.name2executable = {} + self.executable2info = {} + self.hook = hook + + def get_executable(self, envconfig): + """ return path object to the executable for the given + name (e.g. python2.7, python3.6, python etc.) + if name is already an existing path, return name. + If an interpreter cannot be found, return None. + """ + try: + return self.name2executable[envconfig.envname] + except KeyError: + exe = self.hook.tox_get_python_executable(envconfig=envconfig) + reporter.verbosity2("{} uses {}".format(envconfig.envname, exe)) + self.name2executable[envconfig.envname] = exe + return exe + + def get_info(self, envconfig): + executable = self.get_executable(envconfig) + name = envconfig.basepython + if not executable: + return NoInterpreterInfo(name=name) + try: + return self.executable2info[executable] + except KeyError: + info = run_and_get_interpreter_info(name, executable) + self.executable2info[executable] = info + return info + + def get_sitepackagesdir(self, info, envdir): + if not info.executable: + return "" + envdir = str(envdir) + try: + res = exec_on_interpreter(str(info.executable), SITE_PACKAGE_QUERY_SCRIPT, str(envdir)) + except ExecFailed as e: + reporter.verbosity1("execution failed: {} -- {}".format(e.out, e.err)) + return "" + else: + return res["dir"] + + +def run_and_get_interpreter_info(name, executable): + assert executable + try: + result = exec_on_interpreter(str(executable), VERSION_QUERY_SCRIPT) + result["version_info"] = tuple(result["version_info"]) # fix json dump transformation + del result["name"] + del result["version"] + result["executable"] = str(executable) + except ExecFailed as e: + return NoInterpreterInfo(name, executable=e.executable, out=e.out, err=e.err) + else: + return InterpreterInfo(name, **result) + + +def exec_on_interpreter(*args): + from subprocess import Popen, PIPE + + popen = Popen(args, stdout=PIPE, stderr=PIPE, universal_newlines=True) + out, err = popen.communicate() + if popen.returncode: + raise ExecFailed(args[0], args[1:], out, err) + if err: + sys.stderr.write(err) + try: + result = json.loads(out) + except Exception: + raise ExecFailed(args[0], args[1:], out, "could not decode {!r}".format(out)) + return result + + +class ExecFailed(Exception): + def __init__(self, executable, source, out, err): + self.executable = executable + self.source = source + self.out = out + self.err = err + + +class InterpreterInfo: + def __init__(self, name, executable, version_info, sysplatform, is_64): + self.name = name + self.executable = executable + self.version_info = version_info + self.sysplatform = sysplatform + self.is_64 = is_64 + + def __str__(self): + return "".format(self.executable, self.version_info) + + +class NoInterpreterInfo: + def __init__(self, name, executable=None, out=None, err="not found"): + self.name = name + self.executable = executable + self.version_info = None + self.out = out + self.err = err + + def __str__(self): + if self.executable: + return "".format(self.executable) + else: + return "".format(self.name) + + +if tox.INFO.IS_WIN: + from .windows import tox_get_python_executable +else: + from .unix import tox_get_python_executable +assert tox_get_python_executable diff --git a/src/tox/interpreters/py_spec.py b/src/tox/interpreters/py_spec.py new file mode 100644 index 000000000..104af4177 --- /dev/null +++ b/src/tox/interpreters/py_spec.py @@ -0,0 +1,76 @@ +from __future__ import unicode_literals + +import os +import re +import sys + +import six + +import tox + + +class PythonSpec(object): + def __init__(self, name, major, minor, architecture, path, args=None): + self.name = name + self.major = major + self.minor = minor + self.architecture = architecture + self.path = path + self.args = args + + def __repr__(self): + return ( + "{0.__class__.__name__}(name={0.name!r}, major={0.major!r}, minor={0.minor!r}, " + "architecture={0.architecture!r}, path={0.path!r}, args={0.args!r})" + ).format(self) + + def __str__(self): + msg = repr(self) + return msg if six.PY3 else msg.encode("utf-8") + + def satisfies(self, req): + if req.is_abs and self.is_abs and self.path != req.path: + return False + if req.name is not None and req.name != self.name: + return False + if req.architecture is not None and req.architecture != self.architecture: + return False + if req.major is not None and req.major != self.major: + return False + if req.minor is not None and req.minor != self.minor: + return False + if req.major is None and req.minor is not None: + return False + return True + + @property + def is_abs(self): + return self.path is not None and os.path.isabs(self.path) + + @classmethod + def from_name(cls, base_python): + name, major, minor, architecture, path = None, None, None, None, None + if os.path.isabs(base_python): + path = base_python + else: + match = re.match(r"(python|pypy|jython)(\d)?(?:\.(\d))?(-(32|64))?", base_python) + if match: + groups = match.groups() + name = groups[0] + major = int(groups[1]) if len(groups) >= 2 and groups[1] is not None else None + minor = int(groups[2]) if len(groups) >= 3 and groups[2] is not None else None + architecture = ( + int(groups[3]) if len(groups) >= 4 and groups[3] is not None else None + ) + else: + path = base_python + return cls(name, major, minor, architecture, path) + + +CURRENT = PythonSpec( + "pypy" if tox.constants.INFO.IS_PYPY else "python", + sys.version_info[0], + sys.version_info[1], + 64 if sys.maxsize > 2 ** 32 else 32, + sys.executable, +) diff --git a/src/tox/interpreters/unix.py b/src/tox/interpreters/unix.py new file mode 100644 index 000000000..c8738de90 --- /dev/null +++ b/src/tox/interpreters/unix.py @@ -0,0 +1,21 @@ +from __future__ import unicode_literals + +import tox + +from .py_spec import CURRENT, PythonSpec +from .via_path import check_with_path + + +@tox.hookimpl +def tox_get_python_executable(envconfig): + base_python = envconfig.basepython + spec = PythonSpec.from_name(base_python) + # first, check current + if spec.name is not None and CURRENT.satisfies(spec): + return CURRENT.path + # second check if the literal base python + candidates = [base_python] + # third check if the un-versioned name is good + if spec.name is not None and spec.name != base_python: + candidates.append(spec.name) + return check_with_path(candidates, spec) diff --git a/src/tox/interpreters/via_path.py b/src/tox/interpreters/via_path.py new file mode 100644 index 000000000..746eb0651 --- /dev/null +++ b/src/tox/interpreters/via_path.py @@ -0,0 +1,71 @@ +from __future__ import unicode_literals + +import json +import os +import subprocess +from collections import defaultdict +from threading import Lock + +import py + +from tox import reporter +from tox.constants import VERSION_QUERY_SCRIPT + +from .py_spec import PythonSpec + + +def check_with_path(candidates, spec): + for path in candidates: + base = path + if not os.path.isabs(path): + path = py.path.local.sysfind(path) + if path is not None: + if os.path.exists(str(path)): + cur_spec = exe_spec(path, base) + if cur_spec is not None and cur_spec.satisfies(spec): + return cur_spec.path + + +_SPECS = {} +_SPECK_LOCK = defaultdict(Lock) + + +def exe_spec(python_exe, base): + if not isinstance(python_exe, str): + python_exe = str(python_exe) + with _SPECK_LOCK[python_exe]: + if python_exe not in _SPECS: + info = get_python_info([python_exe]) + if info is not None: + found = PythonSpec( + info["name"], + info["version_info"][0], + info["version_info"][1], + 64 if info["is_64"] else 32, + info["executable"], + ) + reporter.verbosity2("{} ({}) is {}".format(base, python_exe, info)) + else: + found = None + _SPECS[python_exe] = found + return _SPECS[python_exe] + + +def get_python_info(cmd): + proc = subprocess.Popen( + cmd + [VERSION_QUERY_SCRIPT], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + out, err = proc.communicate() + if not proc.returncode: + try: + result = json.loads(out) + except ValueError as exception: + failure = exception + else: + return result + else: + failure = "exit code {}".format(proc.returncode) + reporter.verbosity1("{!r} cmd {!r} out {!r} err {!r} ".format(failure, cmd, out, err)) diff --git a/src/tox/interpreters/windows/__init__.py b/src/tox/interpreters/windows/__init__.py new file mode 100644 index 000000000..359a0f8b0 --- /dev/null +++ b/src/tox/interpreters/windows/__init__.py @@ -0,0 +1,52 @@ +from __future__ import unicode_literals + +from threading import Lock + +import tox + +from ..py_spec import CURRENT, PythonSpec +from ..via_path import check_with_path + + +@tox.hookimpl +def tox_get_python_executable(envconfig): + base_python = envconfig.basepython + spec = PythonSpec.from_name(base_python) + # first, check current + if spec.name is not None and CURRENT.satisfies(spec): + return CURRENT.path + + # second check if the py.exe has it (only for non path specs) + if spec.path is None: + py_exe = locate_via_pep514(spec) + if py_exe is not None: + return py_exe + + # third check if the literal base python is on PATH + candidates = [envconfig.basepython] + # fourth check if the name is on PATH + if spec.name is not None and spec.name != base_python: + candidates.append(spec.name) + # or check known locations + if spec.major is not None and spec.minor is not None: + if spec.name == "python": + # The standard names are in predictable places. + candidates.append(r"c:\python{}{}\python.exe".format(spec.major, spec.minor)) + return check_with_path(candidates, spec) + + +_PY_AVAILABLE = [] +_PY_LOCK = Lock() + + +def locate_via_pep514(spec): + with _PY_LOCK: + if not _PY_AVAILABLE: + from . import pep514 + + for spec in pep514.discover_pythons(): + _PY_AVAILABLE.append(spec) + _PY_AVAILABLE.append(CURRENT) + for cur_spec in _PY_AVAILABLE: + if cur_spec.satisfies(spec): + return cur_spec.path diff --git a/src/tox/interpreters/windows/pep514.py b/src/tox/interpreters/windows/pep514.py new file mode 100644 index 000000000..9c7708bf4 --- /dev/null +++ b/src/tox/interpreters/windows/pep514.py @@ -0,0 +1,165 @@ +"""Implement https://www.python.org/dev/peps/pep-0514/ to discover interpreters - Windows only""" +from __future__ import unicode_literals + +import os +import re + +import six +import winreg + +from tox import reporter +from tox.interpreters.py_spec import PythonSpec + + +def enum_keys(key): + at = 0 + while True: + try: + yield winreg.EnumKey(key, at) + except OSError: + break + at += 1 + + +def get_value(key, value_name): + try: + return winreg.QueryValueEx(key, value_name)[0] + except OSError: + return None + + +def discover_pythons(): + for hive, hive_name, key, flags, default_arch in [ + (winreg.HKEY_CURRENT_USER, "HKEY_CURRENT_USER", r"Software\Python", 0, 64), + ( + winreg.HKEY_LOCAL_MACHINE, + "HKEY_LOCAL_MACHINE", + r"Software\Python", + winreg.KEY_WOW64_64KEY, + 64, + ), + ( + winreg.HKEY_LOCAL_MACHINE, + "HKEY_LOCAL_MACHINE", + r"Software\Python", + winreg.KEY_WOW64_32KEY, + 32, + ), + ]: + for spec in process_set(hive, hive_name, key, flags, default_arch): + yield spec + + +def process_set(hive, hive_name, key, flags, default_arch): + try: + with winreg.OpenKeyEx(hive, key, access=winreg.KEY_READ | flags) as root_key: + for company in enum_keys(root_key): + if company == "PyLauncher": # reserved + continue + for spec in process_company(hive_name, company, root_key, default_arch): + yield spec + except OSError: + pass + + +def process_company(hive_name, company, root_key, default_arch): + with winreg.OpenKeyEx(root_key, company) as company_key: + for tag in enum_keys(company_key): + for spec in process_tag(hive_name, company, company_key, tag, default_arch): + yield spec + + +def process_tag(hive_name, company, company_key, tag, default_arch): + with winreg.OpenKeyEx(company_key, tag) as tag_key: + major, minor = load_version_data(hive_name, company, tag, tag_key) + if major is None: + return + arch = load_arch_data(hive_name, company, tag, tag_key, default_arch) + exe, args = load_exe(hive_name, company, company_key, tag) + if exe is not None: + name = "python" if company == "PythonCore" else company + yield PythonSpec(name, major, minor, arch, exe, args) + + +def load_exe(hive_name, company, company_key, tag): + key_path = "{}/{}/{}".format(hive_name, company, tag) + try: + with winreg.OpenKeyEx(company_key, r"{}\InstallPath".format(tag)) as ip_key: + with ip_key: + exe = get_value(ip_key, "ExecutablePath") + if exe is None: + ip = get_value(ip_key, None) + if ip is None: + msg(key_path, "no ExecutablePath or default for it") + + else: + exe = os.path.join(ip, "python.exe") + if os.path.exists(exe): + args = get_value(ip_key, "ExecutableArguments") + return exe, args + else: + msg(key_path, "exe does not exists {}".format(key_path, exe)) + except OSError: + msg("{}/{}".format(key_path, "InstallPath"), "missing") + return None, None + + +def load_arch_data(hive_name, company, tag, tag_key, default_arch): + arch_str = get_value(tag_key, "SysArchitecture") + if arch_str is not None: + key_path = "{}/{}/{}/SysArchitecture".format(hive_name, company, tag) + try: + return parse_arch(arch_str) + except ValueError as sys_arch: + msg(key_path, sys_arch) + return default_arch + + +def parse_arch(arch_str): + if not isinstance(arch_str, six.string_types): + raise ValueError("arch is not string") + match = re.match(r"(\d+)bit", arch_str) + if match: + return int(next(iter(match.groups()))) + raise ValueError("invalid format {}".format(arch_str)) + + +def load_version_data(hive_name, company, tag, tag_key): + version_str = get_value(tag_key, "SysVersion") + major, minor = None, None + if version_str is not None: + key_path = "{}/{}/{}/SysVersion".format(hive_name, company, tag) + try: + major, minor = parse_version(get_value(tag_key, "SysVersion")) + except ValueError as sys_version: + msg(key_path, sys_version) + if major is None: + key_path = "{}/{}/{}".format(hive_name, company, tag) + try: + major, minor = parse_version(tag) + except ValueError as tag_version: + msg(key_path, tag_version) + return major, minor + + +def parse_version(version_str): + if not isinstance(version_str, six.string_types): + raise ValueError("key is not string") + match = re.match(r"(\d+)\.(\d+).*", version_str) + if match: + return tuple(int(i) for i in match.groups()) + raise ValueError("invalid format {}".format(version_str)) + + +def msg(path, what): + reporter.verbosity1("PEP-514 violation in Windows Registry at {} error: {}".format(path, what)) + + +def _run(): + reporter.update_default_reporter(0, reporter.Verbosity.DEBUG) + for spec in discover_pythons(): + print(repr(spec)) + + +if __name__ == "__main__": + _run() diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 09f5bf5c3..b814b7a6e 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -1993,7 +1993,6 @@ def test_default_factors_conflict_ignore(self, newconfig, capsys): assert envconfig.basepython == "python2.7" assert len(record) == 0, "\n".join(repr(r.message) for r in record) - @pytest.mark.issue188 def test_factors_in_boolean(self, newconfig): inisource = """ [tox] @@ -2007,7 +2006,6 @@ def test_factors_in_boolean(self, newconfig): assert configs["py27"].recreate assert not configs["py36"].recreate - @pytest.mark.issue190 def test_factors_in_setenv(self, newconfig): inisource = """ [tox] @@ -2021,7 +2019,6 @@ def test_factors_in_setenv(self, newconfig): assert configs["py27"].setenv["X"] == "1" assert "X" not in configs["py36"].setenv - @pytest.mark.issue191 def test_factor_use_not_checked(self, newconfig): inisource = """ [tox] @@ -2033,7 +2030,6 @@ def test_factor_use_not_checked(self, newconfig): configs = newconfig([], inisource).envconfigs assert set(configs.keys()) == {"py27-a", "py27-b"} - @pytest.mark.issue198 def test_factors_groups_touch(self, newconfig): inisource = """ [tox] diff --git a/tests/unit/test_interpreters.py b/tests/unit/interpreters/test_interpreters.py similarity index 92% rename from tests/unit/test_interpreters.py rename to tests/unit/interpreters/test_interpreters.py index e7928cf75..ea6f65dff 100644 --- a/tests/unit/test_interpreters.py +++ b/tests/unit/interpreters/test_interpreters.py @@ -10,7 +10,6 @@ import tox from tox import reporter -from tox._pytestplugin import mark_dont_run_on_posix from tox.config import get_plugin_manager from tox.interpreters import ( ExecFailed, @@ -29,20 +28,6 @@ def create_interpreters_instance(): return Interpreters(hook=pm.hook) -@mark_dont_run_on_posix -def test_locate_via_py(monkeypatch): - import tox.interpreters - - spec = tox.interpreters.CURRENT - del tox.interpreters._PY_AVAILABLE[:] - exe = tox.interpreters.locate_via_py(spec) - assert exe - assert len(tox.interpreters._PY_AVAILABLE) - - monkeypatch.setattr(tox.interpreters, "_call_py", None) - assert tox.interpreters.locate_via_py(spec) - - def test_tox_get_python_executable(): class envconfig: basepython = sys.executable @@ -93,7 +78,7 @@ def test_find_alias_on_path(monkeypatch, tmp_path): os.symlink(sys.executable, str(magic)) monkeypatch.setenv( str("PATH"), - os.pathsep.join(([str(tmp_path)] + os.environ.get(str("PATH"), "").split(os.pathsep))), + os.pathsep.join([str(tmp_path)] + os.environ.get(str("PATH"), "").split(os.pathsep)), ) class envconfig: diff --git a/tests/unit/interpreters/windows/test_pep514.py b/tests/unit/interpreters/windows/test_pep514.py new file mode 100644 index 000000000..cc97457c2 --- /dev/null +++ b/tests/unit/interpreters/windows/test_pep514.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + +import inspect +import subprocess +import sys + +from tox._pytestplugin import mark_dont_run_on_posix + + +@mark_dont_run_on_posix +def test_discover_winreg(): + from tox.interpreters.windows.pep514 import discover_pythons + + list(discover_pythons()) # raises no error + + +@mark_dont_run_on_posix +def test_run_pep514_main_no_warnings(): + # check we trigger no warnings + import tox.interpreters.windows.pep514 as pep514 + + out = subprocess.check_output( + [sys.executable, inspect.getsourcefile(pep514)], universal_newlines=True + ) + assert "PEP-514 violation in Windows Registry " not in out, out diff --git a/tests/unit/interpreters/windows/test_windows.py b/tests/unit/interpreters/windows/test_windows.py new file mode 100644 index 000000000..43cb7ccad --- /dev/null +++ b/tests/unit/interpreters/windows/test_windows.py @@ -0,0 +1,20 @@ +from tox._pytestplugin import mark_dont_run_on_posix + + +@mark_dont_run_on_posix +def test_locate_via_pep514(monkeypatch): + from tox.interpreters.py_spec import CURRENT + import tox.interpreters.windows + + del tox.interpreters.windows._PY_AVAILABLE[:] + exe = tox.interpreters.windows.locate_via_pep514(CURRENT) + assert exe + assert len(tox.interpreters.windows._PY_AVAILABLE) + + import tox.interpreters.windows.pep514 + + def raise_on_call(): + raise RuntimeError() + + monkeypatch.setattr(tox.interpreters.windows.pep514, "discover_pythons", raise_on_call) + assert tox.interpreters.windows.locate_via_pep514(CURRENT) diff --git a/tests/unit/session/test_show_config.py b/tests/unit/session/test_show_config.py index 1a01d1d89..9a8bff4c0 100644 --- a/tests/unit/session/test_show_config.py +++ b/tests/unit/session/test_show_config.py @@ -1,6 +1,6 @@ import py import pytest -from six import StringIO +from six import PY2, StringIO from six.moves import configparser @@ -9,7 +9,7 @@ def load_config(args, cmd): result.assert_success(is_run_test_env=False) parser = configparser.ConfigParser() output = StringIO(result.out) - parser.readfp(output) + (parser.readfp if PY2 else parser.read_file)(output) return parser diff --git a/tox.ini b/tox.ini index dc192ae09..d891acf25 100644 --- a/tox.ini +++ b/tox.ini @@ -95,7 +95,7 @@ commands = echo {posargs} [flake8] max-complexity = 22 max-line-length = 99 -ignore = E203, W503, C901, E402 +ignore = E203, W503, C901, E402, B011 [pep8] max-line-length = 99 @@ -129,6 +129,9 @@ rsyncdirs = tests tox looponfailroots = tox tests testpaths = tests xfail_strict = True +markers = + git + network [isort] multi_line_output = 3 @@ -136,7 +139,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml +known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml,winreg [testenv:release] description = do a release, required posarg of the version number From 016e9f2997a6518f3c7bea2b18ea11ea6ab62292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 23 May 2019 13:35:57 +0100 Subject: [PATCH 25/72] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3e1835ccf..ce1b997c0 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,8 @@ much about the checklist - we will help you get started. - [ ] added news fragment in [changelog folder](/docs/changelog) * fragment name: `..rst` for example (588.bugfix.rst) * `` is must be one of `bugfix`, `feature`, `deprecation`,`breaking`, `doc`, `misc` - * if pr has no issue: consider creating one first or change it to the pr number after creating the pr - * "sign" fragment with "by @" + * if PR has no issue: consider creating one first or change it to the PR number after creating the PR + * "sign" fragment with "by :user:``" * please use full sentences with correct case and punctuation, for example: "Fix issue with non-ascii contents in doctest text files - by :user:`superuser`." * also see [examples](/docs/changelog/examples.rst) - [ ] added yourself to `CONTRIBUTORS` (preserving alphabetical order) From c7456aa2636b30bdf2f09b05dc19cd9ac1424f42 Mon Sep 17 00:00:00 2001 From: Florian Schulze Date: Thu, 23 May 2019 16:15:39 +0200 Subject: [PATCH 26/72] Fix for --result-json with --parallel (#1309) * Correct ``--result-json`` output with ``--parallel``. (#1295) When using ``--parallel`` with ``--result-json`` the test results are now included the same way as with serial runs. This is accomplished by generating json result output for each individual run and at the end copy the data into the main json result output. * avoid duplication in code, improve coverage --- docs/changelog/1184.feature.rst | 2 +- docs/changelog/1295.bugfix.rst | 1 + src/tox/constants.py | 2 + src/tox/session/__init__.py | 34 +++++-- src/tox/session/commands/run/parallel.py | 3 + src/tox/util/lock.py | 2 +- src/tox/venv.py | 11 +++ tests/unit/session/test_parallel.py | 108 +++++++++++++++++++---- tests/unit/test_z_cmdline.py | 2 +- 9 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 docs/changelog/1295.bugfix.rst diff --git a/docs/changelog/1184.feature.rst b/docs/changelog/1184.feature.rst index 736a7a581..388f3f5a5 100644 --- a/docs/changelog/1184.feature.rst +++ b/docs/changelog/1184.feature.rst @@ -1 +1 @@ -Adding ```TOX_PARALLEL_NO_SPINNER``` environment variable to disable the spinner in parallel mode for the purposes of clean output when using CI tools - by :user:`zeroshift` +Adding ``TOX_PARALLEL_NO_SPINNER`` environment variable to disable the spinner in parallel mode for the purposes of clean output when using CI tools - by :user:`zeroshift` diff --git a/docs/changelog/1295.bugfix.rst b/docs/changelog/1295.bugfix.rst new file mode 100644 index 000000000..c201f127a --- /dev/null +++ b/docs/changelog/1295.bugfix.rst @@ -0,0 +1 @@ +When using ``--parallel`` with ``--result-json`` the test results are now included the same way as with serial runs - by :user:`fschulze` diff --git a/src/tox/constants.py b/src/tox/constants.py index a2c2ed157..8dcc86bf8 100644 --- a/src/tox/constants.py +++ b/src/tox/constants.py @@ -86,3 +86,5 @@ class PIP: SITE_PACKAGE_QUERY_SCRIPT = os.path.join(_HELP_DIR, "get_site_package_dir.py") BUILD_REQUIRE_SCRIPT = os.path.join(_HELP_DIR, "build_requires.py") BUILD_ISOLATED = os.path.join(_HELP_DIR, "build_isolated.py") +PARALLEL_RESULT_JSON_PREFIX = ".tox-result" +PARALLEL_RESULT_JSON_SUFFIX = ".json" diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py index de885c8f1..e93886841 100644 --- a/src/tox/session/__init__.py +++ b/src/tox/session/__init__.py @@ -4,7 +4,9 @@ setup by using virtualenv. Configuration is generally done through an INI-style "tox.ini" file. """ +from __future__ import absolute_import, unicode_literals +import json import os import re import subprocess @@ -220,6 +222,24 @@ def subcommand_test(self): retcode = self._summary() return retcode + def _add_parallel_summaries(self): + if self.config.option.parallel != PARALLEL_OFF and "testenvs" in self.resultlog.dict: + result_log = self.resultlog.dict["testenvs"] + for tox_env in self.venv_dict.values(): + data = self._load_parallel_env_report(tox_env) + if data and "testenvs" in data and tox_env.name in data["testenvs"]: + result_log[tox_env.name] = data["testenvs"][tox_env.name] + + @staticmethod + def _load_parallel_env_report(tox_env): + """Load report data into memory, remove disk file""" + result_json_path = tox_env.get_result_json_path() + if result_json_path and result_json_path.exists(): + with result_json_path.open("r") as file_handler: + data = json.load(file_handler) + result_json_path.remove() + return data + def _summary(self): is_parallel_child = PARALLEL_ENV_VAR_KEY in os.environ if not is_parallel_child: @@ -254,12 +274,14 @@ def _summary(self): report(msg) if not exit_code and not is_parallel_child: reporter.good(" congratulations :)") - if not is_parallel_child: - path = self.config.option.resultjson - if path: - path = py.path.local(path) - path.write(self.resultlog.dumps_json()) - reporter.line("wrote json report at: {}".format(path)) + path = self.config.option.resultjson + if path: + if not is_parallel_child: + self._add_parallel_summaries() + path = py.path.local(path) + data = self.resultlog.dumps_json() + reporter.line("write json report at: {}".format(path)) + path.write(data) return exit_code def showconfig(self): diff --git a/src/tox/session/commands/run/parallel.py b/src/tox/session/commands/run/parallel.py index e307c1cfd..76db97933 100644 --- a/src/tox/session/commands/run/parallel.py +++ b/src/tox/session/commands/run/parallel.py @@ -42,6 +42,9 @@ def run_in_thread(tox_env, os_env, processes): if hasattr(tox_env, "package"): args_sub.insert(position, str(tox_env.package)) args_sub.insert(position, "--installpkg") + if tox_env.get_result_json_path(): + result_json_index = args_sub.index("--result-json") + args_sub[result_json_index + 1] = "{}".format(tox_env.get_result_json_path()) with tox_env.new_action("parallel {}".format(tox_env.name)) as action: def collect_process(process): diff --git a/src/tox/util/lock.py b/src/tox/util/lock.py index 00c86b5e7..fd6473407 100644 --- a/src/tox/util/lock.py +++ b/src/tox/util/lock.py @@ -36,6 +36,6 @@ def get_unique_file(path, prefix, suffix): max_value = max(max_value, int(candidate.basename[len(prefix) : -len(suffix)])) except ValueError: continue - winner = path.join("{}{}.log".format(prefix, max_value + 1)) + winner = path.join("{}{}{}".format(prefix, max_value + 1, suffix)) winner.ensure(dir=0) return winner diff --git a/src/tox/venv.py b/src/tox/venv.py index 1f7860433..7ad2d00f2 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -13,7 +13,9 @@ from tox import reporter from tox.action import Action from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY +from tox.constants import PARALLEL_RESULT_JSON_PREFIX, PARALLEL_RESULT_JSON_SUFFIX from tox.package.local import resolve_package +from tox.util.lock import get_unique_file from tox.util.path import ensure_empty_dir from .config import DepConfig @@ -113,6 +115,7 @@ def __init__(self, envconfig=None, popen=None, env_log=None): self.popen = popen self._actions = [] self.env_log = env_log + self._result_json_path = None def new_action(self, msg, *args): config = self.envconfig.config @@ -130,6 +133,14 @@ def new_action(self, msg, *args): self.envconfig.envpython, ) + def get_result_json_path(self): + if self._result_json_path is None: + if self.envconfig.config.option.resultjson: + self._result_json_path = get_unique_file( + self.path, PARALLEL_RESULT_JSON_PREFIX, PARALLEL_RESULT_JSON_SUFFIX + ) + return self._result_json_path + @property def hook(self): return self.envconfig.config.pluginmanager.hook diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index 3d8746b50..fcb51a212 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -1,10 +1,16 @@ from __future__ import absolute_import, unicode_literals +import json +import os +import subprocess import sys +import threading import pytest from flaky import flaky +from tox._pytestplugin import RunResult + def test_parallel(cmd, initproj): initproj( @@ -26,7 +32,7 @@ def test_parallel(cmd, initproj): """, }, ) - result = cmd("--parallel", "all") + result = cmd("-p", "all") result.assert_success() @@ -49,7 +55,7 @@ def test_parallel_live(cmd, initproj): """, }, ) - result = cmd("--parallel", "all", "--parallel-live") + result = cmd("-p", "all", "-o") result.assert_success() @@ -73,7 +79,7 @@ def test_parallel_circular(cmd, initproj): """, }, ) - result = cmd("--parallel", "1") + result = cmd("-p", "1") result.assert_fail() assert result.out == "ERROR: circular dependency detected: a | b\n" @@ -191,26 +197,96 @@ def test_parallel_show_output(cmd, initproj, monkeypatch): assert "stderr always" in result.out, result.output() -def test_parallel_no_spinner(cmd, initproj, monkeypatch): - monkeypatch.setenv(str("TOX_PARALLEL_NO_SPINNER"), str("1")) - initproj( +@pytest.fixture() +def parallel_project(initproj): + return initproj( "pkg123-0.7", filedefs={ "tox.ini": """ [tox] + skipsdist = True envlist = a, b - isolated_build = true [testenv] + skip_install = True commands=python -c "import sys; print(sys.executable)" - [testenv:b] - depends = a - """, - "pyproject.toml": """ - [build-system] - requires = ["setuptools >= 35.0.2"] - build-backend = 'setuptools.build_meta' - """, + """ }, ) - result = cmd("--parallel", "all") + + +def test_parallel_no_spinner_on(cmd, parallel_project, monkeypatch): + monkeypatch.setenv(str("TOX_PARALLEL_NO_SPINNER"), str("1")) + result = cmd("-p", "all") + result.assert_success() + assert "[2] a | b" not in result.out + + +def test_parallel_no_spinner_off(cmd, parallel_project, monkeypatch): + monkeypatch.setenv(str("TOX_PARALLEL_NO_SPINNER"), str("0")) + result = cmd("-p", "all") result.assert_success() + assert "[2] a | b" in result.out + + +def test_parallel_no_spinner_not_set(cmd, parallel_project, monkeypatch): + monkeypatch.delenv(str("TOX_PARALLEL_NO_SPINNER"), raising=False) + result = cmd("-p", "all") + result.assert_success() + assert "[2] a | b" in result.out + + +def test_parallel_result_json(cmd, parallel_project, tmp_path): + parallel_result_json = tmp_path / "parallel.json" + result = cmd("-p", "all", "--result-json", "{}".format(parallel_result_json)) + ensure_result_json_ok(result, parallel_result_json) + + +def ensure_result_json_ok(result, json_path): + if isinstance(result, RunResult): + result.assert_success() + else: + assert not isinstance(result, subprocess.CalledProcessError) + assert json_path.exists() + serial_data = json.loads(json_path.read_text()) + ensure_key_in_env(serial_data) + + +def ensure_key_in_env(serial_data): + for env in ("a", "b"): + for key in ("setup", "test"): + assert key in serial_data["testenvs"][env], json.dumps( + serial_data["testenvs"], indent=2 + ) + + +def test_parallel_result_json_concurrent(cmd, parallel_project, tmp_path): + # first run to set up the environments (env creation is not thread safe) + result = cmd("-p", "all") + result.assert_success() + + invoke_result = {} + + def invoke_tox_in_thread(thread_name, result_json): + try: + # needs to be process to have it's own stdout + invoke_result[thread_name] = subprocess.check_output( + [sys.executable, "-m", "tox", "-p", "all", "--result-json", str(result_json)], + universal_newlines=True, + ) + except subprocess.CalledProcessError as exception: + invoke_result[thread_name] = exception + + # now concurrently + parallel1_result_json = tmp_path / "parallel1.json" + parallel2_result_json = tmp_path / "parallel2.json" + threads = [ + threading.Thread(target=invoke_tox_in_thread, args=(k, p)) + for k, p in (("t1", parallel1_result_json), ("t2", parallel2_result_json)) + ] + [t.start() for t in threads] + [t.join() for t in threads] + + ensure_result_json_ok(invoke_result["t1"], parallel1_result_json) + ensure_result_json_ok(invoke_result["t2"], parallel2_result_json) + # our set_os_env_var is not thread-safe so clean-up TOX_WORK_DIR + os.environ.pop("TOX_WORK_DIR", None) diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index ed889809e..60941971c 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -500,7 +500,7 @@ def test_result_json(cmd, initproj, example123): assert isinstance(pyinfo["version_info"], list) assert pyinfo["version"] assert pyinfo["executable"] - assert "wrote json report at: {}".format(json_path) == result.outlines[-1] + assert "write json report at: {}".format(json_path) == result.outlines[-1] def test_developz(initproj, cmd): From 772a43678f97274f817e9590165037456e1eb1d7 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 23 May 2019 15:19:32 +0100 Subject: [PATCH 27/72] release 3.12.0 --- docs/changelog.rst | 19 +++++++++++++++++++ docs/changelog/1184.feature.rst | 1 - docs/changelog/1295.bugfix.rst | 1 - docs/changelog/1306.bugfix.rst | 1 - 4 files changed, 19 insertions(+), 3 deletions(-) delete mode 100644 docs/changelog/1184.feature.rst delete mode 100644 docs/changelog/1295.bugfix.rst delete mode 100644 docs/changelog/1306.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 690389ab3..d316e66c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,25 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.12.0 (2019-05-23) +-------------------- + +Bugfixes +^^^^^^^^ + +- When using ``--parallel`` with ``--result-json`` the test results are now included the same way as with serial runs - by :user:`fschulze` + `#1295 `_ +- Turns out the output of the ``py -0p`` is not stable yet and varies depending on various edge cases. Instead now we read the interpreter values directly from registry via `PEP-514 `_ - by :user:`gaborbernat`. + `#1306 `_ + + +Features +^^^^^^^^ + +- Adding ``TOX_PARALLEL_NO_SPINNER`` environment variable to disable the spinner in parallel mode for the purposes of clean output when using CI tools - by :user:`zeroshift` + `#1184 `_ + + v3.11.1 (2019-05-16) -------------------- diff --git a/docs/changelog/1184.feature.rst b/docs/changelog/1184.feature.rst deleted file mode 100644 index 388f3f5a5..000000000 --- a/docs/changelog/1184.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Adding ``TOX_PARALLEL_NO_SPINNER`` environment variable to disable the spinner in parallel mode for the purposes of clean output when using CI tools - by :user:`zeroshift` diff --git a/docs/changelog/1295.bugfix.rst b/docs/changelog/1295.bugfix.rst deleted file mode 100644 index c201f127a..000000000 --- a/docs/changelog/1295.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -When using ``--parallel`` with ``--result-json`` the test results are now included the same way as with serial runs - by :user:`fschulze` diff --git a/docs/changelog/1306.bugfix.rst b/docs/changelog/1306.bugfix.rst deleted file mode 100644 index 469ca1635..000000000 --- a/docs/changelog/1306.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Turns out the output of the ``py -0p`` is not stable yet and varies depending on various edge cases. Instead now we read the interpreter values directly from registry via `PEP-514 `_ - by :user:`gaborbernat`. From 1c74a510562afdd5428f003e54cf4394bad59513 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 23 May 2019 09:43:22 -0700 Subject: [PATCH 28/72] Ensure native strings in environ (#1314) --- docs/changelog/1313.bugfix.rst | 1 + src/tox/session/__init__.py | 2 +- tests/unit/session/test_session.py | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1313.bugfix.rst diff --git a/docs/changelog/1313.bugfix.rst b/docs/changelog/1313.bugfix.rst new file mode 100644 index 000000000..8b89489dd --- /dev/null +++ b/docs/changelog/1313.bugfix.rst @@ -0,0 +1 @@ +Ensure ``TOX_WORK_DIR`` is a native string in ``os.environ`` - by :user:`asottile`. diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py index e93886841..048feb94e 100644 --- a/src/tox/session/__init__.py +++ b/src/tox/session/__init__.py @@ -63,7 +63,7 @@ def main(args): try: config = load_config(args) config.logdir.ensure(dir=1) - with set_os_env_var("TOX_WORK_DIR", config.toxworkdir): + with set_os_env_var(str("TOX_WORK_DIR"), config.toxworkdir): session = build_session(config) exit_code = session.runcommand() if exit_code is None: diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index b90430395..df5814bb6 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -285,6 +285,10 @@ def assert_popen_env(res): if tox_id != "GLOB": assert env["TOX_ENV_NAME"] == tox_id assert env["TOX_ENV_DIR"] == os.path.join(res.cwd, ".tox", tox_id) + # ensure native strings for environ for windows + for k, v in env.items(): + assert type(k) is str, (k, v, type(k)) + assert type(v) is str, (k, v, type(v)) def test_command_prev_post_ok(cmd, initproj, mock_venv): From ca49c94723ec9a55fa742190a32f43b1c78dc4d3 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Thu, 23 May 2019 10:16:46 -0700 Subject: [PATCH 29/72] fix winreg usage for python2 on windows (#1316) --- docs/changelog/1315.bugfix.rst | 1 + src/tox/interpreters/windows/pep514.py | 4 ++-- tox.ini | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/1315.bugfix.rst diff --git a/docs/changelog/1315.bugfix.rst b/docs/changelog/1315.bugfix.rst new file mode 100644 index 000000000..2861120c4 --- /dev/null +++ b/docs/changelog/1315.bugfix.rst @@ -0,0 +1 @@ +Fix import and usage of ``winreg`` for python2.7 on windows - by :user:`asottile`. diff --git a/src/tox/interpreters/windows/pep514.py b/src/tox/interpreters/windows/pep514.py index 9c7708bf4..2396262a1 100644 --- a/src/tox/interpreters/windows/pep514.py +++ b/src/tox/interpreters/windows/pep514.py @@ -5,7 +5,7 @@ import re import six -import winreg +from six.moves import winreg from tox import reporter from tox.interpreters.py_spec import PythonSpec @@ -52,7 +52,7 @@ def discover_pythons(): def process_set(hive, hive_name, key, flags, default_arch): try: - with winreg.OpenKeyEx(hive, key, access=winreg.KEY_READ | flags) as root_key: + with winreg.OpenKeyEx(hive, key, 0, winreg.KEY_READ | flags) as root_key: for company in enum_keys(root_key): if company == "PyLauncher": # reserved continue diff --git a/tox.ini b/tox.ini index d891acf25..21fac4d99 100644 --- a/tox.ini +++ b/tox.ini @@ -139,7 +139,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml,winreg +known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml [testenv:release] description = do a release, required posarg of the version number From 5813770ab9440bef0a5328458b90f96844cad9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 23 May 2019 18:21:23 +0100 Subject: [PATCH 30/72] fix windows selects incorrect spec (#1317) --- docs/changelog/1317.bugfix.rst | 1 + src/tox/interpreters/windows/__init__.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/1317.bugfix.rst diff --git a/docs/changelog/1317.bugfix.rst b/docs/changelog/1317.bugfix.rst new file mode 100644 index 000000000..bcedc5460 --- /dev/null +++ b/docs/changelog/1317.bugfix.rst @@ -0,0 +1 @@ +Fix Windows selects incorrect spec on first discovery - by :user:`gaborbernat` diff --git a/src/tox/interpreters/windows/__init__.py b/src/tox/interpreters/windows/__init__.py index 359a0f8b0..3933ae18a 100644 --- a/src/tox/interpreters/windows/__init__.py +++ b/src/tox/interpreters/windows/__init__.py @@ -44,8 +44,7 @@ def locate_via_pep514(spec): if not _PY_AVAILABLE: from . import pep514 - for spec in pep514.discover_pythons(): - _PY_AVAILABLE.append(spec) + _PY_AVAILABLE.extend(pep514.discover_pythons()) _PY_AVAILABLE.append(CURRENT) for cur_spec in _PY_AVAILABLE: if cur_spec.satisfies(spec): From 3ddd0e5f859155a8b4a537a53c75c5c383423382 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Thu, 23 May 2019 18:42:16 +0100 Subject: [PATCH 31/72] release 3.12.1 --- docs/changelog.rst | 14 ++++++++++++++ docs/changelog/1313.bugfix.rst | 1 - docs/changelog/1315.bugfix.rst | 1 - docs/changelog/1317.bugfix.rst | 1 - 4 files changed, 14 insertions(+), 3 deletions(-) delete mode 100644 docs/changelog/1313.bugfix.rst delete mode 100644 docs/changelog/1315.bugfix.rst delete mode 100644 docs/changelog/1317.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index d316e66c3..7c17891d0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,20 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.12.1 (2019-05-23) +-------------------- + +Bugfixes +^^^^^^^^ + +- Ensure ``TOX_WORK_DIR`` is a native string in ``os.environ`` - by :user:`asottile`. + `#1313 `_ +- Fix import and usage of ``winreg`` for python2.7 on windows - by :user:`asottile`. + `#1315 `_ +- Fix Windows selects incorrect spec on first discovery - by :user:`gaborbernat` + `#1317 `_ + + v3.12.0 (2019-05-23) -------------------- diff --git a/docs/changelog/1313.bugfix.rst b/docs/changelog/1313.bugfix.rst deleted file mode 100644 index 8b89489dd..000000000 --- a/docs/changelog/1313.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure ``TOX_WORK_DIR`` is a native string in ``os.environ`` - by :user:`asottile`. diff --git a/docs/changelog/1315.bugfix.rst b/docs/changelog/1315.bugfix.rst deleted file mode 100644 index 2861120c4..000000000 --- a/docs/changelog/1315.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix import and usage of ``winreg`` for python2.7 on windows - by :user:`asottile`. diff --git a/docs/changelog/1317.bugfix.rst b/docs/changelog/1317.bugfix.rst deleted file mode 100644 index bcedc5460..000000000 --- a/docs/changelog/1317.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix Windows selects incorrect spec on first discovery - by :user:`gaborbernat` From 8c0275f36a9cf1f560733022008c5782bd9b4eb2 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 27 May 2019 23:34:35 -0700 Subject: [PATCH 32/72] Clean up parser options now that we're using argparse (#1325) --- src/tox/config/__init__.py | 49 ++++++-------------------------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 6551e0cd1..d2fb4d82e 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -377,14 +377,9 @@ def __setitem__(self, name, value): @tox.hookimpl def tox_addoption(parser): parser.add_argument( - "--version", - action="store_true", - dest="version", - help="report version information to stdout.", - ) - parser.add_argument( - "-h", "--help", action="store_true", dest="help", help="show help about options" + "--version", action="store_true", help="report version information to stdout." ) + parser.add_argument("-h", "--help", action="store_true", help="show help about options") parser.add_argument( "--help-ini", "--hi", action="store_true", dest="helpini", help="show help about ini-names" ) @@ -399,22 +394,16 @@ def tox_addoption(parser): "-l", "--listenvs", action="store_true", - dest="listenvs", help="show list of test environments (with description if verbose)", ) parser.add_argument( "-a", "--listenvs-all", action="store_true", - dest="listenvs_all", help="show list of all defined environments (with description if verbose)", ) parser.add_argument( - "-c", - action="store", - default=None, - dest="configfile", - help="config file name or directory with 'tox.ini' file.", + "-c", dest="configfile", help="config file name or directory with 'tox.ini' file." ) parser.add_argument( "-e", @@ -423,14 +412,9 @@ def tox_addoption(parser): metavar="envlist", help="work against specified environments (ALL selects all).", ) + parser.add_argument("--notest", action="store_true", help="skip invoking test commands.") parser.add_argument( - "--notest", action="store_true", dest="notest", help="skip invoking test commands." - ) - parser.add_argument( - "--sdistonly", - action="store_true", - dest="sdistonly", - help="only perform the sdist packaging activity.", + "--sdistonly", action="store_true", help="only perform the sdist packaging activity." ) add_parallel_flags(parser) parser.add_argument( @@ -442,15 +426,12 @@ def tox_addoption(parser): ) parser.add_argument( "--installpkg", - action="store", - default=None, metavar="PATH", help="use specified package for installation into venv, instead of creating an sdist.", ) parser.add_argument( "--develop", action="store_true", - dest="develop", help="install package in the venv using 'setup.py develop' via 'pip -e .'", ) parser.add_argument( @@ -465,21 +446,15 @@ def tox_addoption(parser): parser.add_argument( "--pre", action="store_true", - dest="pre", help="install pre-releases and development versions of dependencies. " "This will pass the --pre option to install_command " "(pip by default).", ) parser.add_argument( - "-r", - "--recreate", - action="store_true", - dest="recreate", - help="force recreation of virtual environments", + "-r", "--recreate", action="store_true", help="force recreation of virtual environments" ) parser.add_argument( "--result-json", - action="store", dest="resultjson", metavar="PATH", help="write a json file with detailed information " @@ -489,9 +464,7 @@ def tox_addoption(parser): # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument( "--hashseed", - action="store", metavar="SEED", - default=None, help="set PYTHONHASHSEED to SEED before running commands. " "Defaults to a random integer in the range [1, 4294967295] " "([1, 1024] on Windows). " @@ -501,7 +474,6 @@ def tox_addoption(parser): "--force-dep", action="append", metavar="REQ", - default=None, help="Forces a certain version of one of the dependencies " "when configuring the virtual environment. REQ Examples " "'pytest<2.7' or 'django>=1.6'.", @@ -516,14 +488,7 @@ def tox_addoption(parser): ) cli_skip_missing_interpreter(parser) - parser.add_argument( - "--workdir", - action="store", - dest="workdir", - metavar="PATH", - default=None, - help="tox working directory", - ) + parser.add_argument("--workdir", metavar="PATH", help="tox working directory") parser.add_argument( "args", nargs="*", help="additional arguments available to command positional substitution" From afe11da3e339434d2778b5d702fc8e676dc4f54a Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 28 May 2019 08:26:50 -0700 Subject: [PATCH 33/72] Move to importlib-metadata for performance improvements (#1324) --- docs/changelog/1324.feature.rst | 1 + setup.cfg | 5 +++-- src/tox/config/__init__.py | 26 +++++++++++++++---------- src/tox/package/builder/isolated.py | 7 ++++--- src/tox/package/local.py | 7 +++---- src/tox/session/commands/show_config.py | 21 +++++++++++++------- src/tox/venv.py | 3 +-- tests/unit/session/test_provision.py | 2 +- tox.ini | 2 +- 9 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 docs/changelog/1324.feature.rst diff --git a/docs/changelog/1324.feature.rst b/docs/changelog/1324.feature.rst new file mode 100644 index 000000000..38551c06f --- /dev/null +++ b/docs/changelog/1324.feature.rst @@ -0,0 +1 @@ +Replace ``pkg_resources`` with ``importlib_metadata`` for speed - by :user:`asottile`. diff --git a/setup.cfg b/setup.cfg index 569f4ca2e..d5f3143c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,8 +38,9 @@ classifiers = packages = find: python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* install_requires = - setuptools >= 30.0.0 - pluggy >= 0.3.0, <1 + importlib-metadata + packaging + pluggy >= 0.12.0, <1 py >= 1.4.17, <2 six >= 1.0.0, <2 virtualenv >= 14.0.0 diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index d2fb4d82e..211f24196 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -15,13 +15,16 @@ from subprocess import list2cmdline from threading import Thread -import pkg_resources +import importlib_metadata import pluggy import py import toml +from packaging import requirements +from packaging.utils import canonicalize_name import tox from tox.constants import INFO +from tox.exception import MissingDependency from tox.interpreters import Interpreters, NoInterpreterInfo from tox.reporter import ( REPORTER_TIMESTAMP_ON_ENV, @@ -192,10 +195,10 @@ def _cut_off_dep_comment(name): @classmethod def _is_same_dep(cls, dep1, dep2): """Definitions are the same if they refer to the same package, even if versions differ.""" - dep1_name = pkg_resources.Requirement.parse(dep1).project_name + dep1_name = canonicalize_name(requirements.Requirement(dep1).name) try: - dep2_name = pkg_resources.Requirement.parse(dep2).project_name - except pkg_resources.RequirementParseError: + dep2_name = canonicalize_name(requirements.Requirement(dep2).name) + except requirements.InvalidRequirement: # we couldn't parse a version, probably a URL return False return dep1_name == dep2_name @@ -1133,17 +1136,20 @@ def ensure_requires_satisfied(config, requires, min_version): for require in requires + [min_version]: # noinspection PyBroadException try: - package = pkg_resources.Requirement.parse(require) - if package.project_name not in exists: + package = requirements.Requirement(require) + package_name = canonicalize_name(package.name) + if package_name not in exists: deps.append(DepConfig(require, None)) - exists.add(package.project_name) - pkg_resources.get_distribution(package) - except pkg_resources.RequirementParseError as exception: + exists.add(package_name) + dist = importlib_metadata.distribution(package_name) + if not package.specifier.contains(dist.version, prereleases=True): + raise MissingDependency(package) + except requirements.InvalidRequirement as exception: failed_to_parse = True error("failed to parse {!r}".format(exception)) except Exception as exception: verbosity1("could not satisfy requires {!r}".format(exception)) - missing_requirements.append(str(pkg_resources.Requirement(require))) + missing_requirements.append(str(requirements.Requirement(require))) if failed_to_parse: raise tox.exception.BadRequirement() config.run_provision = bool(len(missing_requirements)) diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index 9519fb0c8..99b5a83f9 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -3,8 +3,9 @@ import json from collections import namedtuple -import pkg_resources import six +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from tox import reporter from tox.config import DepConfig, get_py_project_toml @@ -31,11 +32,11 @@ def build(config, session): build_requires = get_build_requires(build_info, package_venv, config.setupdir) # we need to filter out requirements already specified in pyproject.toml or user deps - base_build_deps = {pkg_resources.Requirement(r.name).key for r in package_venv.envconfig.deps} + base_build_deps = {canonicalize_name(r.name) for r in package_venv.envconfig.deps} build_requires_dep = [ DepConfig(r, None) for r in build_requires - if pkg_resources.Requirement(r).key not in base_build_deps + if canonicalize_name(Requirement(r).name) not in base_build_deps ] if build_requires_dep: with package_venv.new_action("build_requires", package_venv.envconfig.envdir) as action: diff --git a/src/tox/package/local.py b/src/tox/package/local.py index b56e31cfa..aa0751b9e 100644 --- a/src/tox/package/local.py +++ b/src/tox/package/local.py @@ -1,7 +1,7 @@ import os import re -import pkg_resources +import packaging.version import py import tox @@ -58,7 +58,6 @@ def get_version_from_filename(basename): return None version = m.group(1) try: - - return pkg_resources.packaging.version.Version(version) - except pkg_resources.packaging.version.InvalidVersion: + return packaging.version.Version(version) + except packaging.version.InvalidVersion: return None diff --git a/src/tox/session/commands/show_config.py b/src/tox/session/commands/show_config.py index f307cc186..11c5cf267 100644 --- a/src/tox/session/commands/show_config.py +++ b/src/tox/session/commands/show_config.py @@ -1,6 +1,9 @@ import sys from collections import OrderedDict +import importlib_metadata +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from six import StringIO from six.moves import configparser @@ -57,17 +60,21 @@ def tox_info(config, parser): def version_info(parser): - import pkg_resources - versions = OrderedDict() - visited = set() to_visit = {"tox"} while to_visit: current = to_visit.pop() - visited.add(current) - current_dist = pkg_resources.get_distribution(current) - to_visit.update(i.name for i in current_dist.requires() if i.name not in visited) - versions[current] = current_dist.version + current_dist = importlib_metadata.distribution(current) + current_name = canonicalize_name(current_dist.metadata["name"]) + versions[current_name] = current_dist.version + if current_dist.requires is not None: + for require in current_dist.requires: + pkg = Requirement(require) + pkg_name = canonicalize_name(pkg.name) + if ( + pkg.marker is None or pkg.marker.evaluate({"extra": ""}) + ) and pkg_name not in versions: + to_visit.add(pkg_name) set_section(parser, "tox:versions", versions) diff --git a/src/tox/venv.py b/src/tox/venv.py index 7ad2d00f2..9d0c8a5ab 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -7,7 +7,6 @@ from itertools import chain import py -from pkg_resources import to_filename import tox from tox import reporter @@ -324,7 +323,7 @@ def _needs_reinstall(self, setupdir, action): sys_path = json.loads(out) except ValueError: sys_path = [] - egg_info_fname = ".".join((to_filename(name), "egg-info")) + egg_info_fname = ".".join((name.replace("-", "_"), "egg-info")) for d in reversed(sys_path): egg_info = py.path.local(d).join(egg_info_fname) if egg_info.check(): diff --git a/tests/unit/session/test_provision.py b/tests/unit/session/test_provision.py index 6408dc654..83b992f41 100644 --- a/tests/unit/session/test_provision.py +++ b/tests/unit/session/test_provision.py @@ -95,7 +95,7 @@ def test_provision_bad_requires(newconfig, capsys, monkeypatch): """, ) out, err = capsys.readouterr() - assert "ERROR: failed to parse RequirementParseError" in out + assert "ERROR: failed to parse InvalidRequirement" in out assert not err diff --git a/tox.ini b/tox.ini index 21fac4d99..4194f2f65 100644 --- a/tox.ini +++ b/tox.ini @@ -139,7 +139,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pkg_resources,pluggy,py,pytest,setuptools,six,sphinx,toml +known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,importlib_metadata,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,six,sphinx,toml [testenv:release] description = do a release, required posarg of the version number From 69bad039b6eea6666c3b06ad62b2730a3a933cd7 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 28 May 2019 08:32:30 -0700 Subject: [PATCH 34/72] Fix unrecognized arguments error (#1327) --- src/tox/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 211f24196..f42f60088 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -125,7 +125,7 @@ def add_testenv_attribute_obj(self, obj): def parse_cli(self, args, strict=False): args, argv = self.argparser.parse_known_args(args) if argv and (strict or WITHIN_PROVISION): - self.argparser.error("unrecognized arguments: %s".format(" ".join(argv))) + self.argparser.error("unrecognized arguments: {}".format(" ".join(argv))) return args def _format_help(self): From 7237c9d84bc2fe25a324a1488b34f497d86aac0c Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 28 May 2019 08:38:40 -0700 Subject: [PATCH 35/72] Add bounds for importlib-metadata / packaging (#1328) --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d5f3143c2..583e90fd6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,8 +38,8 @@ classifiers = packages = find: python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* install_requires = - importlib-metadata - packaging + importlib-metadata >= 0.12, <1 + packaging >= 14 pluggy >= 0.12.0, <1 py >= 1.4.17, <2 six >= 1.0.0, <2 From 1554104e0877421b749febd73a577c9c69aed8ea Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 1 Jun 2019 08:51:20 -0700 Subject: [PATCH 36/72] Implement --devenv (#1326) --- docs/changelog/1326.feature.rst | 1 + docs/example/devenv.rst | 36 +++++++++++++++++++++++--- src/tox/config/__init__.py | 19 +++++++++++++- tests/unit/test_z_cmdline.py | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/1326.feature.rst diff --git a/docs/changelog/1326.feature.rst b/docs/changelog/1326.feature.rst new file mode 100644 index 000000000..19a5e3052 --- /dev/null +++ b/docs/changelog/1326.feature.rst @@ -0,0 +1 @@ +Add the ``--devenv ENVDIR`` option for creating development environments from ``[testenv]`` configurations - by :user:`asottile`. diff --git a/docs/example/devenv.rst b/docs/example/devenv.rst index 2d62987b9..57e9ffaaf 100644 --- a/docs/example/devenv.rst +++ b/docs/example/devenv.rst @@ -10,16 +10,44 @@ environments. It can also be used for setting up normalized project development environments and thus help reduce the risk of different team members using mismatched development environments. + +Creating development environments using the ``--devenv`` option +=============================================================== + +The easiest way to set up a development environment is to use the ``--devenv`` +option along with your existing configured ``testenv``s. The ``--devenv`` +option accepts a single argument, the location you want to create a development +environment at. + +For example, if I wanted to replicate the ``py36`` environment, I could run:: + + $ tox --devenv venv-py36 -e py36 + ... + $ source venv-py36/bin/activate + (venv-py36) $ python --version + Python 3.6.7 + +The ``--devenv`` option skips the ``commands=`` section of that configured +test environment and always sets ``usedevelop=true`` for the environment that +is created. + +If you don't specify an environment with ``-e``, the devenv feature will +default to ``-e py`` -- usually taking the interpreter you're running ``tox`` +with and the default ``[testenv]`` configuration. + +Creating development environments using configuration +===================================================== + Here are some examples illustrating how to set up a project's development environment using tox. For illustration purposes, let us call the development environment ``dev``. Example 1: Basic scenario -========================= +------------------------- Step 1 - Configure the development environment ----------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ First, we prepare the tox configuration for our development environment by defining a ``[testenv:dev]`` section in the project's ``tox.ini`` @@ -54,7 +82,7 @@ configuration: Step 2 - Create the development environment -------------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Once the ``[testenv:dev]`` configuration section has been defined, we create the actual development environment by running the following: @@ -68,7 +96,7 @@ This creates the environment at the path specified by the environment's Example 2: A more complex scenario -================================== +---------------------------------- Let us say we want our project development environment to: diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index f42f60088..dc57f775e 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -415,6 +415,9 @@ def tox_addoption(parser): metavar="envlist", help="work against specified environments (ALL selects all).", ) + parser.add_argument( + "--devenv", help="sets up a development environment based on the tox configuration." + ) parser.add_argument("--notest", action="store_true", help="skip invoking test commands.") parser.add_argument( "--sdistonly", action="store_true", help="only perform the sdist packaging activity." @@ -497,12 +500,19 @@ def tox_addoption(parser): "args", nargs="*", help="additional arguments available to command positional substitution" ) + def _set_envdir_from_devenv(testenv_config, value): + if testenv_config.config.option.devenv: + return py.path.local(testenv_config.config.option.devenv) + else: + return value + parser.add_testenv_attribute( name="envdir", type="path", default="{toxworkdir}/{envname}", help="set venv directory -- be very careful when changing this as tox " "will remove this directory when recreating an environment", + postprocess=_set_envdir_from_devenv, ) # add various core venv interpreter attributes @@ -751,7 +761,7 @@ def pip_pre(testenv_config, value): def develop(testenv_config, value): option = testenv_config.config.option - return not option.installpkg and (value or option.develop) + return not option.installpkg and (value or option.develop or bool(option.devenv)) parser.add_testenv_attribute( name="usedevelop", @@ -1106,6 +1116,12 @@ def run(name, section, subs, config): config.skipsdist = reader.getbool("skipsdist", all_develop) + if config.option.devenv: + config.option.notest = True + + if config.option.devenv and len(config.envlist) != 1: + feedback("--devenv requires only a single -e", sysexit=True) + def handle_provision(self, config, reader): requires_list = reader.getlist("requires") config.minversion = reader.getstring("minversion", None) @@ -1251,6 +1267,7 @@ def _getenvdata(self, reader, config): (os.environ.get(PARALLEL_ENV_VAR_KEY), True), (from_option, True), (from_environ, True), + ("py" if self.config.option.devenv else None, False), (from_config, False), ) env_str, envlist_explicit = next(((i, e) for i, e in candidates if i), ([], False)) diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index 60941971c..f193df4dc 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -789,6 +789,52 @@ def test_notest_setup_py_error(initproj, cmd): assert re.search("ERROR:.*InvocationError", result.out) +def test_devenv(initproj, cmd): + initproj( + "example123", + filedefs={ + "setup.py": """\ + from setuptools import setup + setup(name='x') + """, + "tox.ini": """\ + [tox] + # envlist is ignored for --devenv + envlist = foo,bar,baz + + [testenv] + # --devenv implies --notest + commands = python -c "exit(1)" + """, + }, + ) + result = cmd("--devenv", "venv") + result.assert_success() + # `--devenv` defaults to the `py` environment and a develop install + assert "py develop-inst:" in result.out + assert re.search("py create:.*venv", result.out) + + +def test_devenv_does_not_allow_multiple_environments(initproj, cmd): + initproj( + "example123", + filedefs={ + "setup.py": """\ + from setuptools import setup + setup(name='x') + """, + "tox.ini": """\ + [tox] + envlist=foo,bar,baz + """, + }, + ) + + result = cmd("--devenv", "venv", "-e", "foo,bar") + result.assert_fail() + assert result.err == "ERROR: --devenv requires only a single -e\n" + + def test_PYC(initproj, cmd, monkeypatch): initproj("example123", filedefs={"tox.ini": ""}) monkeypatch.setenv("PYTHONDOWNWRITEBYTECODE", "1") From bed76e60ac965fad1868049d0126eaddb21b991d Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 2 Jun 2019 00:13:17 -0700 Subject: [PATCH 37/72] Improve --devenv help and ensure `--devenv ` does not delete (#1333) --- src/tox/config/__init__.py | 17 +++++++++++------ tests/unit/test_z_cmdline.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index dc57f775e..2ef07a463 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -416,7 +416,12 @@ def tox_addoption(parser): help="work against specified environments (ALL selects all).", ) parser.add_argument( - "--devenv", help="sets up a development environment based on the tox configuration." + "--devenv", + metavar="ENVDIR", + help=( + "sets up a development environment at ENVDIR based on the env's tox " + "configuration specified by `-e` (-e defaults to py)." + ), ) parser.add_argument("--notest", action="store_true", help="skip invoking test commands.") parser.add_argument( @@ -501,7 +506,7 @@ def tox_addoption(parser): ) def _set_envdir_from_devenv(testenv_config, value): - if testenv_config.config.option.devenv: + if testenv_config.config.option.devenv is not None: return py.path.local(testenv_config.config.option.devenv) else: return value @@ -761,7 +766,7 @@ def pip_pre(testenv_config, value): def develop(testenv_config, value): option = testenv_config.config.option - return not option.installpkg and (value or option.develop or bool(option.devenv)) + return not option.installpkg and (value or option.develop or option.devenv is not None) parser.add_testenv_attribute( name="usedevelop", @@ -1116,10 +1121,10 @@ def run(name, section, subs, config): config.skipsdist = reader.getbool("skipsdist", all_develop) - if config.option.devenv: + if config.option.devenv is not None: config.option.notest = True - if config.option.devenv and len(config.envlist) != 1: + if config.option.devenv is not None and len(config.envlist) != 1: feedback("--devenv requires only a single -e", sysexit=True) def handle_provision(self, config, reader): @@ -1267,7 +1272,7 @@ def _getenvdata(self, reader, config): (os.environ.get(PARALLEL_ENV_VAR_KEY), True), (from_option, True), (from_environ, True), - ("py" if self.config.option.devenv else None, False), + ("py" if self.config.option.devenv is not None else None, False), (from_config, False), ) env_str, envlist_explicit = next(((i, e) for i, e in candidates if i), ([], False)) diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index f193df4dc..53a4e50c6 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -835,6 +835,27 @@ def test_devenv_does_not_allow_multiple_environments(initproj, cmd): assert result.err == "ERROR: --devenv requires only a single -e\n" +def test_devenv_does_not_delete_project(initproj, cmd): + initproj( + "example123", + filedefs={ + "setup.py": """\ + from setuptools import setup + setup(name='x') + """, + "tox.ini": """\ + [tox] + envlist=foo,bar,baz + """, + }, + ) + + result = cmd("--devenv", "") + result.assert_fail() + assert "would delete project" in result.out + assert "ERROR: ConfigError: envdir must not equal toxinidir" in result.out + + def test_PYC(initproj, cmd, monkeypatch): initproj("example123", filedefs={"tox.ini": ""}) monkeypatch.setenv("PYTHONDOWNWRITEBYTECODE", "1") From bc317225bc8c7107a5b54d592bc3600516605633 Mon Sep 17 00:00:00 2001 From: Oliver Bestwalter Date: Wed, 5 Jun 2019 15:08:24 +0200 Subject: [PATCH 38/72] envconfig.missing_subs is not a setting: should be a private attribute (#1335) --- src/tox/config/__init__.py | 4 ++-- src/tox/venv.py | 4 ++-- tests/unit/config/test_config.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 2ef07a463..19820d033 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -883,7 +883,7 @@ def __init__(self, envname, config, factors, reader): #: set of factors self.factors = factors self._reader = reader - self.missing_subs = [] + self._missing_subs = [] """Holds substitutions that could not be resolved. Pre 2.8.1 missing substitutions crashed with a ConfigError although this would not be a @@ -1233,7 +1233,7 @@ def make_envconfig(self, name, section, subs, config, replace=True): if env_attr.postprocess: res = env_attr.postprocess(testenv_config=tc, value=res) except tox.exception.MissingSubstitution as e: - tc.missing_subs.append(e.name) + tc._missing_subs.append(e.name) res = e.FLAG setattr(tc, env_attr.name, res) if atype in ("path", "string", "basepython"): diff --git a/src/tox/venv.py b/src/tox/venv.py index 9d0c8a5ab..72b0b737f 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -575,11 +575,11 @@ def _pcall( ) def setupenv(self): - if self.envconfig.missing_subs: + if self.envconfig._missing_subs: self.status = ( "unresolvable substitution(s): {}. " "Environment variables are missing or defined recursively.".format( - ",".join(["'{}'".format(m) for m in self.envconfig.missing_subs]) + ",".join(["'{}'".format(m) for m in self.envconfig._missing_subs]) ) ) return diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index b814b7a6e..7c1fa6045 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -556,7 +556,7 @@ def test_missing_env_sub_raises_config_error_in_non_testenv(self, newconfig): def test_missing_env_sub_populates_missing_subs(self, newconfig): config = newconfig("[testenv:foo]\ncommands={env:VAR}") print(SectionReader("section", config._cfg).getstring("commands")) - assert config.envconfigs["foo"].missing_subs == ["VAR"] + assert config.envconfigs["foo"]._missing_subs == ["VAR"] def test_getstring_environment_substitution_with_default(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") From ef843dcde0882bedc3be32c0b12191054b18fffb Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 5 Jun 2019 18:11:36 +0100 Subject: [PATCH 39/72] Use POSIX shell rules for creating posargs string on non-Windows platforms (#1336) * Use POSIX shell rules for creating posargs string on non-Windows platforms. Closes #1336 * pre-commit run --all-files --- CONTRIBUTORS | 1 + docs/changelog/1336.bugfix.rst | 2 ++ src/tox/config/__init__.py | 11 ++++++++++- tests/unit/config/test_config.py | 12 +++++++++++- 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/1336.bugfix.rst diff --git a/CONTRIBUTORS b/CONTRIBUTORS index a95197ffb..39d848b4f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -8,6 +8,7 @@ Anthon van der Neuth Anthony Sottile Ashley Whetter Asmund Grammeltwedt +Barney Gale Barry Warsaw Bartolome Sanchez Salado Benoit Pierre diff --git a/docs/changelog/1336.bugfix.rst b/docs/changelog/1336.bugfix.rst new file mode 100644 index 000000000..17f9dd556 --- /dev/null +++ b/docs/changelog/1336.bugfix.rst @@ -0,0 +1,2 @@ +tox used Windows shell rules on non-Windows platforms when transforming +positional arguments to a string - by :user:`barneygale`. diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 19820d033..4437268e3 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -39,6 +39,12 @@ from .parallel import add_parallel_config, add_parallel_flags from .reporter import add_verbosity_commands +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + + hookimpl = tox.hookimpl """DEPRECATED - REMOVE - this is left for compatibility with plugins importing this from here. @@ -1674,7 +1680,10 @@ def getargvlist(cls, reader, value, replace=True): @classmethod def processcommand(cls, reader, command, replace=True): posargs = getattr(reader, "posargs", "") - posargs_string = list2cmdline([x for x in posargs if x]) + if sys.platform.startswith("win"): + posargs_string = list2cmdline([x for x in posargs if x]) + else: + posargs_string = " ".join([shlex_quote(x) for x in posargs if x]) # Iterate through each word of the command substituting as # appropriate to construct the new command string. This diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 7c1fa6045..2f019b791 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -730,8 +730,18 @@ def test_argvlist_posargs_with_quotes(self, newconfig): cmd1 -f {posargs} """ ) + # The operating system APIs for launching processes differ between + # Windows and other OSs. On Windows, the command line is passed as a + # string (and not a list of strings). Python uses the MS C runtime + # rules for splitting this string into `sys.argv`, and those rules + # differ from POSIX shell rules in their treatment of quoted arguments. + if sys.platform.startswith("win"): + substitutions = ["foo", "'bar", "baz'"] + else: + substitutions = ["foo", "bar baz"] + reader = SectionReader("section", config._cfg) - reader.addsubstitutions(["foo", "'bar", "baz'"]) + reader.addsubstitutions(substitutions) assert reader.getargvlist("key1") == [] x = reader.getargvlist("key2") assert x == [["cmd1", "-f", "foo", "bar baz"]] From 81d168332dfe1bd6fdeeb0bd740077ed3a379993 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 10 Jun 2019 08:10:50 -0700 Subject: [PATCH 40/72] cowardly refuse to delete a directory if it isn't a venv (#1340) --- docs/changelog/1340.feature.rst | 1 + src/tox/venv.py | 27 ++++++++++++++++++++++++++- tests/unit/test_z_cmdline.py | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1340.feature.rst diff --git a/docs/changelog/1340.feature.rst b/docs/changelog/1340.feature.rst new file mode 100644 index 000000000..3c37d416a --- /dev/null +++ b/docs/changelog/1340.feature.rst @@ -0,0 +1 @@ +Refuse to delete ``envdir`` if it doesn't look like a virtualenv - by :user:`asottile`. diff --git a/src/tox/venv.py b/src/tox/venv.py index 72b0b737f..1bd71e6db 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -12,7 +12,7 @@ from tox import reporter from tox.action import Action from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY -from tox.constants import PARALLEL_RESULT_JSON_PREFIX, PARALLEL_RESULT_JSON_SUFFIX +from tox.constants import INFO, PARALLEL_RESULT_JSON_PREFIX, PARALLEL_RESULT_JSON_SUFFIX from tox.package.local import resolve_package from tox.util.lock import get_unique_file from tox.util.path import ensure_empty_dir @@ -694,6 +694,31 @@ def tox_testenv_create(venv, action): def cleanup_for_venv(venv): within_parallel = PARALLEL_ENV_VAR_KEY in os.environ + # if the directory exists and it doesn't look like a virtualenv, produce + # an error + if venv.path.exists(): + dir_items = set(os.listdir(str(venv.path))) - {".lock", "log"} + dir_items = {p for p in dir_items if not p.startswith(".tox-")} + else: + dir_items = set() + + if not ( + # doesn't exist => OK + not venv.path.exists() + # does exist, but it's empty => OK + or not dir_items + # it exists and we're on windows with Lib and Scripts => OK + or (INFO.IS_WIN and dir_items > {"Scripts", "Lib"}) + # non-windows, with lib and bin => OK + or dir_items > {"bin", "lib"} + ): + venv.status = "error" + reporter.error( + "cowardly refusing to delete `envdir` (it does not look like a virtualenv): " + "{}".format(venv.path) + ) + raise SystemExit(2) + if within_parallel: if venv.path.exists(): # do not delete the log folder as that's used by parent diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index 53a4e50c6..ce34a07dc 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -115,6 +115,26 @@ def test_envdir_equals_toxini_errors_out(cmd, initproj): result.assert_fail() +def test_envdir_would_delete_some_directory(cmd, initproj): + projdir = initproj( + "example-123", + filedefs={ + "tox.ini": """\ + [tox] + + [testenv:venv] + envdir=example + commands= + """ + }, + ) + + result = cmd("-e", "venv") + assert projdir.join("example/__init__.py").exists() + result.assert_fail() + assert "cowardly refusing to delete `envdir`" in result.out + + def test_run_custom_install_command_error(cmd, initproj): initproj( "interp123-0.5", From 520c7ae8723949c9e60444a8114dcaa0626b4abb Mon Sep 17 00:00:00 2001 From: Oliver Bestwalter Date: Wed, 12 Jun 2019 10:21:48 +0200 Subject: [PATCH 41/72] Fix deprecation comment - recommendation is to import directly (#1341) --- src/tox/config/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 4437268e3..c72296372 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -46,12 +46,9 @@ hookimpl = tox.hookimpl -"""DEPRECATED - REMOVE - this is left for compatibility with plugins importing this from here. +"""DEPRECATED - REMOVE - left for compatibility with plugins importing from here. -Instead create a hookimpl in your code with: - - import pluggy - hookimpl = pluggy.HookimplMarker("tox") +Import hookimpl directly from tox instead. """ default_factors = tox.PYTHON.DEFAULT_FACTORS From 0b76b110c38dfd17aae8c787f9109ff2a6a60127 Mon Sep 17 00:00:00 2001 From: Oliver Bestwalter Date: Fri, 21 Jun 2019 21:59:35 +0200 Subject: [PATCH 42/72] fix #1342 add note that directory is created if does not exist yet (#1345) Also fixed some surrounding typos. --- docs/config.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 90b3d5cb7..546bba18d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -320,7 +320,7 @@ Complete list of settings that you can put into ``testenv*`` sections: .. conf:: whitelist_externals ^ MULTI-LINE-LIST - each line specifies a command name (in glob-style pattern format) + Each line specifies a command name (in glob-style pattern format) which can be used in the ``commands`` section without triggering a "not installed in virtualenv" warning. Example: if you use the unix ``make`` for running tests you can list ``whitelist_externals=make`` @@ -330,7 +330,11 @@ Complete list of settings that you can put into ``testenv*`` sections: .. conf:: changedir ^ PATH ^ {toxinidir} - change to this working directory when executing the test command. + Change to this working directory when executing the test command. + + .. note:: + + If the directory does not exist yet, it will be created. .. conf:: deps ^ MULTI-LINE-LIST From 80e8a096af617b4658a68fb9eb37064092898497 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 24 Jun 2019 18:31:25 +0100 Subject: [PATCH 43/72] release 3.13.0 --- docs/changelog.rst | 22 ++++++++++++++++++++++ docs/changelog/1324.feature.rst | 1 - docs/changelog/1326.feature.rst | 1 - docs/changelog/1336.bugfix.rst | 2 -- docs/changelog/1340.feature.rst | 1 - 5 files changed, 22 insertions(+), 5 deletions(-) delete mode 100644 docs/changelog/1324.feature.rst delete mode 100644 docs/changelog/1326.feature.rst delete mode 100644 docs/changelog/1336.bugfix.rst delete mode 100644 docs/changelog/1340.feature.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c17891d0..9ebd342d3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,28 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.13.0 (2019-06-24) +-------------------- + +Bugfixes +^^^^^^^^ + +- tox used Windows shell rules on non-Windows platforms when transforming + positional arguments to a string - by :user:`barneygale`. + `#1336 `_ + + +Features +^^^^^^^^ + +- Replace ``pkg_resources`` with ``importlib_metadata`` for speed - by :user:`asottile`. + `#1324 `_ +- Add the ``--devenv ENVDIR`` option for creating development environments from ``[testenv]`` configurations - by :user:`asottile`. + `#1326 `_ +- Refuse to delete ``envdir`` if it doesn't look like a virtualenv - by :user:`asottile`. + `#1340 `_ + + v3.12.1 (2019-05-23) -------------------- diff --git a/docs/changelog/1324.feature.rst b/docs/changelog/1324.feature.rst deleted file mode 100644 index 38551c06f..000000000 --- a/docs/changelog/1324.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Replace ``pkg_resources`` with ``importlib_metadata`` for speed - by :user:`asottile`. diff --git a/docs/changelog/1326.feature.rst b/docs/changelog/1326.feature.rst deleted file mode 100644 index 19a5e3052..000000000 --- a/docs/changelog/1326.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add the ``--devenv ENVDIR`` option for creating development environments from ``[testenv]`` configurations - by :user:`asottile`. diff --git a/docs/changelog/1336.bugfix.rst b/docs/changelog/1336.bugfix.rst deleted file mode 100644 index 17f9dd556..000000000 --- a/docs/changelog/1336.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -tox used Windows shell rules on non-Windows platforms when transforming -positional arguments to a string - by :user:`barneygale`. diff --git a/docs/changelog/1340.feature.rst b/docs/changelog/1340.feature.rst deleted file mode 100644 index 3c37d416a..000000000 --- a/docs/changelog/1340.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Refuse to delete ``envdir`` if it doesn't look like a virtualenv - by :user:`asottile`. From d02049579f9f70d4bfb5fe37d3432ec076519e47 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 25 Jun 2019 14:22:56 -0700 Subject: [PATCH 44/72] Fix double requirement in isolated build (#1350) --- docs/changelog/1349.bugfix.rst | 1 + src/tox/package/builder/isolated.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1349.bugfix.rst diff --git a/docs/changelog/1349.bugfix.rst b/docs/changelog/1349.bugfix.rst new file mode 100644 index 000000000..e41baa736 --- /dev/null +++ b/docs/changelog/1349.bugfix.rst @@ -0,0 +1 @@ +Fix isolated build double-requirement - by :user:`asottile`. diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index 99b5a83f9..eda10f2d2 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -32,7 +32,9 @@ def build(config, session): build_requires = get_build_requires(build_info, package_venv, config.setupdir) # we need to filter out requirements already specified in pyproject.toml or user deps - base_build_deps = {canonicalize_name(r.name) for r in package_venv.envconfig.deps} + base_build_deps = { + canonicalize_name(Requirement(r.name).name) for r in package_venv.envconfig.deps + } build_requires_dep = [ DepConfig(r, None) for r in build_requires From e2000ef2f241c57b4ec8cf247f38d75a108c4e67 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 25 Jun 2019 14:57:49 -0700 Subject: [PATCH 45/72] release 3.13.1 --- docs/changelog.rst | 10 ++++++++++ docs/changelog/1349.bugfix.rst | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 docs/changelog/1349.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ebd342d3..5be6a56cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,16 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.13.1 (2019-06-25) +-------------------- + +Bugfixes +^^^^^^^^ + +- Fix isolated build double-requirement - by :user:`asottile`. + `#1349 `_ + + v3.13.0 (2019-06-24) -------------------- diff --git a/docs/changelog/1349.bugfix.rst b/docs/changelog/1349.bugfix.rst deleted file mode 100644 index e41baa736..000000000 --- a/docs/changelog/1349.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix isolated build double-requirement - by :user:`asottile`. From 4639b6c3e80091cc369f13ff9ee9c7772a174e9b Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 25 Jun 2019 14:59:26 -0700 Subject: [PATCH 46/72] Fix bash code blocks in HOWTORELEASE.rst (#1351) --- HOWTORELEASE.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/HOWTORELEASE.rst b/HOWTORELEASE.rst index 8e00edb96..02061f185 100644 --- a/HOWTORELEASE.rst +++ b/HOWTORELEASE.rst @@ -31,9 +31,9 @@ Release ------- Run the release command and make sure you pass in the desired release number: -```bash -tox -e release -- -``` +.. code-block:: bash + + tox -e release -- Create a pull request and wait until it the CI passes. Now make sure you merge the PR and delete the release branch. The CI will automatically pick the tag up and @@ -47,8 +47,8 @@ Make sure to let the world know that a new version is out by whatever means you As a minimum, send out a mail notification by triggering the notify tox environment: -```bash -TOX_DEV_GOOGLE_SECRET=our_secret tox -e notify -``` +.. code-block:: bash + + TOX_DEV_GOOGLE_SECRET=our_secret tox -e notify Note you'll need the ``TOX_DEV_GOOGLE_SECRET`` key, what you can acquire from other maintainers. From 2d2b2aed942937fa11b2e9df191c2542fdcbf89a Mon Sep 17 00:00:00 2001 From: Oliver Bestwalter Date: Fri, 28 Jun 2019 02:47:40 +0200 Subject: [PATCH 47/72] Add explicit check for pypy venv (#1356) * Add explicit check for pypy venv * Add changelog entry * Add regression test for --recreate --- docs/changelog/1355.bugfix.rst | 1 + src/tox/venv.py | 2 ++ tests/unit/test_z_cmdline.py | 6 ++++++ 3 files changed, 9 insertions(+) create mode 100644 docs/changelog/1355.bugfix.rst diff --git a/docs/changelog/1355.bugfix.rst b/docs/changelog/1355.bugfix.rst new file mode 100644 index 000000000..252b5da6c --- /dev/null +++ b/docs/changelog/1355.bugfix.rst @@ -0,0 +1 @@ +on venv cleanup: add explicit check for pypy venv to make it possible to recreate it - by :user:`obestwalter` diff --git a/src/tox/venv.py b/src/tox/venv.py index 1bd71e6db..607f8a84a 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -711,6 +711,8 @@ def cleanup_for_venv(venv): or (INFO.IS_WIN and dir_items > {"Scripts", "Lib"}) # non-windows, with lib and bin => OK or dir_items > {"bin", "lib"} + # pypy has a different lib folder => OK + or dir_items > {"bin", "lib_pypy"} ): venv.status = "error" reporter.error( diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index ce34a07dc..ae74648aa 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -135,6 +135,12 @@ def test_envdir_would_delete_some_directory(cmd, initproj): assert "cowardly refusing to delete `envdir`" in result.out +def test_recreate(cmd, initproj): + initproj("example-123", filedefs={"tox.ini": ""}) + cmd("-e", "py", "--notest").assert_success() + cmd("-r", "-e", "py", "--notest").assert_success() + + def test_run_custom_install_command_error(cmd, initproj): initproj( "interp123-0.5", From e2ae954a89d21ba0916e82c1d983b8f00257324b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 1 Jul 2019 18:00:18 +0100 Subject: [PATCH 48/72] fix non-canonical names don't work with provisioning (#1360) --- .pre-commit-config.yaml | 8 +-- docs/changelog/1359.bugfix.rst | 1 + setup.cfg | 2 +- src/tox/config/__init__.py | 7 +- tests/unit/session/test_provision.py | 95 ++++++++++++++++++++++++++++ tox.ini | 6 +- 6 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/1359.bugfix.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cc63bdc72..386e655ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,18 +6,18 @@ repos: args: [--safe] language_version: python3.7 - repo: https://github.com/asottile/blacken-docs - rev: v0.5.0 + rev: v1.1.0 hooks: - id: blacken-docs additional_dependencies: [black==19.3b0] language_version: python3.7 - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.0 + rev: v1.9.1 hooks: - id: seed-isort-config args: [--application-directories, "src:."] - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.20 + rev: v4.3.21 hooks: - id: isort - repo: https://github.com/pre-commit/pre-commit-hooks @@ -31,7 +31,7 @@ repos: additional_dependencies: ["flake8-bugbear == 19.3.0"] language_version: python3.7 - repo: https://github.com/asottile/pyupgrade - rev: v1.17.1 + rev: v1.19.0 hooks: - id: pyupgrade - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/docs/changelog/1359.bugfix.rst b/docs/changelog/1359.bugfix.rst new file mode 100644 index 000000000..e07a094d6 --- /dev/null +++ b/docs/changelog/1359.bugfix.rst @@ -0,0 +1 @@ +non canonical names within :conf:`requires` cause infinite provisioning loop - by :user:`gaborbernat` diff --git a/setup.cfg b/setup.cfg index 583e90fd6..48dab494a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,7 @@ console_scripts = testing = freezegun >= 0.3.11, <1 pathlib2 >= 2.3.3, <3 - pytest >= 3.0.0, <5 + pytest >= 4.0.0, <6 pytest-cov >= 2.5.1, <3 pytest-mock >= 1.10.0, <2 pytest-xdist >= 1.22.2, <2 diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index c72296372..c1a3239ef 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -1027,7 +1027,7 @@ def line_of_default_to_zero(section, name=None): config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") within_parallel = PARALLEL_ENV_VAR_KEY in os.environ - if not within_parallel: + if not within_parallel and not WITHIN_PROVISION: ensure_empty_dir(config.logdir) # determine indexserver dictionary @@ -1165,7 +1165,7 @@ def ensure_requires_satisfied(config, requires, min_version): if package_name not in exists: deps.append(DepConfig(require, None)) exists.add(package_name) - dist = importlib_metadata.distribution(package_name) + dist = importlib_metadata.distribution(package.name) if not package.specifier.contains(dist.version, prereleases=True): raise MissingDependency(package) except requirements.InvalidRequirement as exception: @@ -1176,6 +1176,9 @@ def ensure_requires_satisfied(config, requires, min_version): missing_requirements.append(str(requirements.Requirement(require))) if failed_to_parse: raise tox.exception.BadRequirement() + if WITHIN_PROVISION and missing_requirements: + msg = "break infinite loop provisioning within {} missing {}" + raise tox.exception.Error(msg.format(sys.executable, missing_requirements)) config.run_provision = bool(len(missing_requirements)) return deps diff --git a/tests/unit/session/test_provision.py b/tests/unit/session/test_provision.py index 83b992f41..e8f7abc23 100644 --- a/tests/unit/session/test_provision.py +++ b/tests/unit/session/test_provision.py @@ -1,9 +1,15 @@ +from __future__ import absolute_import, unicode_literals + +import os import shutil import subprocess import sys import py import pytest +from pathlib2 import Path +from six.moves.urllib.parse import urljoin +from six.moves.urllib.request import pathname2url from tox.exception import BadRequirement, MissingRequirement @@ -141,3 +147,92 @@ def test_provision_cli_args_not_ignored_if_provision_false(cmd, initproj): initproj("test-0.1", {"tox.ini": "[tox]"}) result = cmd("-a", "--option", "b") result.assert_fail(is_run_test_env=False) + + +@pytest.fixture(scope="session") +def wheel(tmp_path_factory): + """create a wheel for a project""" + state = {"at": 0} + + def _wheel(path): + state["at"] += 1 + dest_path = tmp_path_factory.mktemp("wheel-{}-".format(state["at"])) + env = os.environ.copy() + try: + subprocess.check_output( + [ + sys.executable, + "-m", + "pip", + "wheel", + "-w", + str(dest_path), + "--no-deps", + str(path), + ], + universal_newlines=True, + stderr=subprocess.STDOUT, + env=env, + ) + except subprocess.CalledProcessError as exception: + assert not exception.returncode, exception.output + + wheels = list(dest_path.glob("*.whl")) + assert len(wheels) == 1 + wheel = wheels[0] + return wheel + + return _wheel + + +THIS_PROJECT_ROOT = Path(__file__).resolve().parents[3] + + +@pytest.fixture(scope="session") +def tox_wheel(wheel): + return wheel(THIS_PROJECT_ROOT) + + +@pytest.fixture(scope="session") +def magic_non_canonical_wheel(wheel, tmp_path_factory): + magic_proj = tmp_path_factory.mktemp("magic") + (magic_proj / "setup.py").write_text( + "from setuptools import setup\nsetup(name='com.magic.this-is-fun')" + ) + return wheel(magic_proj) + + +def test_provision_non_canonical_dep( + cmd, initproj, monkeypatch, tox_wheel, magic_non_canonical_wheel +): + initproj( + "w-0.1", + { + "tox.ini": """ + [tox] + envlist = py + requires = + com.magic.this-is-fun + tox == {} + [testenv:.tox] + passenv = * + """.format( + tox_wheel.name.split("-")[1] + ) + }, + ) + find_links = " ".join( + space_path2url(d) for d in (tox_wheel.parent, magic_non_canonical_wheel.parent) + ) + + monkeypatch.setenv(str("PIP_FIND_LINKS"), str(find_links)) + + result = cmd("-a", "-v", "-v") + result.assert_success(is_run_test_env=False) + + +def space_path2url(path): + at_path = str(path) + if " " not in at_path: + return at_path + return urljoin("file:", pathname2url(os.path.abspath(at_path))) diff --git a/tox.ini b/tox.ini index 4194f2f65..fa5b321de 100644 --- a/tox.ini +++ b/tox.ini @@ -117,9 +117,9 @@ exclude_lines = [coverage:paths] source = src/tox - .tox/*/lib/python*/site-packages/tox - .tox/pypy*/site-packages/tox - .tox\*\Lib\site-packages\tox + */.tox/*/lib/python*/site-packages/tox + */.tox/pypy*/site-packages/tox + */.tox\*\Lib\site-packages\tox */src/tox *\src\tox From c5ef3d21798d7ec006166343f9631a471d08b551 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 1 Jul 2019 19:14:27 +0100 Subject: [PATCH 49/72] release 3.13.2 --- docs/changelog.rst | 12 ++++++++++++ docs/changelog/1355.bugfix.rst | 1 - docs/changelog/1359.bugfix.rst | 1 - 3 files changed, 12 insertions(+), 2 deletions(-) delete mode 100644 docs/changelog/1355.bugfix.rst delete mode 100644 docs/changelog/1359.bugfix.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index 5be6a56cc..499efce69 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -11,6 +11,18 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +v3.13.2 (2019-07-01) +-------------------- + +Bugfixes +^^^^^^^^ + +- on venv cleanup: add explicit check for pypy venv to make it possible to recreate it - by :user:`obestwalter` + `#1355 `_ +- non canonical names within :conf:`requires` cause infinite provisioning loop - by :user:`gaborbernat` + `#1359 `_ + + v3.13.1 (2019-06-25) -------------------- diff --git a/docs/changelog/1355.bugfix.rst b/docs/changelog/1355.bugfix.rst deleted file mode 100644 index 252b5da6c..000000000 --- a/docs/changelog/1355.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -on venv cleanup: add explicit check for pypy venv to make it possible to recreate it - by :user:`obestwalter` diff --git a/docs/changelog/1359.bugfix.rst b/docs/changelog/1359.bugfix.rst deleted file mode 100644 index e07a094d6..000000000 --- a/docs/changelog/1359.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -non canonical names within :conf:`requires` cause infinite provisioning loop - by :user:`gaborbernat` From a0487dac091adf767e9940a316e3755439661a29 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 2 Jul 2019 16:04:52 +0200 Subject: [PATCH 50/72] Fix repo file links in the PR template (#1363) * Fix repo file links in the PR template * Add a misc changelog fragment --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- docs/changelog/1363.misc.fix-broken-pr-template.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/1363.misc.fix-broken-pr-template.rst diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ce1b997c0..823822f5a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,18 +5,18 @@ much about the checklist - we will help you get started. ## Contribution checklist: -(also see [CONTRIBUTING.rst](/CONTRIBUTING.rst) for details) +(also see [CONTRIBUTING.rst](../tree/master/CONTRIBUTING.rst) for details) - [ ] wrote descriptive pull request text - [ ] added/updated test(s) - [ ] updated/extended the documentation - [ ] added relevant [issue keyword](https://help.github.com/articles/closing-issues-using-keywords/) in message body -- [ ] added news fragment in [changelog folder](/docs/changelog) +- [ ] added news fragment in [changelog folder](../tree/master/docs/changelog) * fragment name: `..rst` for example (588.bugfix.rst) * `` is must be one of `bugfix`, `feature`, `deprecation`,`breaking`, `doc`, `misc` * if PR has no issue: consider creating one first or change it to the PR number after creating the PR * "sign" fragment with "by :user:``" * please use full sentences with correct case and punctuation, for example: "Fix issue with non-ascii contents in doctest text files - by :user:`superuser`." - * also see [examples](/docs/changelog/examples.rst) + * also see [examples](../tree/master/docs/changelog/examples.rst) - [ ] added yourself to `CONTRIBUTORS` (preserving alphabetical order) diff --git a/docs/changelog/1363.misc.fix-broken-pr-template.rst b/docs/changelog/1363.misc.fix-broken-pr-template.rst new file mode 100644 index 000000000..c30e6c4bd --- /dev/null +++ b/docs/changelog/1363.misc.fix-broken-pr-template.rst @@ -0,0 +1 @@ +Fix relative URLs to files in the repo in ``.github/PULL_REQUEST_TEMPLATE.md`` — by :user:`webknjaz` From e4c7752f0a6cb1ce03503a4707209e775da60cb2 Mon Sep 17 00:00:00 2001 From: Marius Gedminas Date: Mon, 8 Jul 2019 18:10:06 +0300 Subject: [PATCH 51/72] Move depends and parallel_show_output into the right section (#1366) `depends` and `parallel_show_output` are not global settings, they're per-environment settings. I think they got listed in the global settings section by accident. --- docs/config.rst | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 546bba18d..3e24c5cf4 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -160,25 +160,6 @@ Global settings are defined under the ``tox`` section as: Name of the virtual environment used to create a source distribution from the source tree. -.. conf:: parallel_show_output ^ bool ^ false - - .. versionadded:: 3.7.0 - - If set to True the content of the output will always be shown when running in parallel mode. - -.. conf:: depends ^ comma separated values - - .. versionadded:: 3.7.0 - - tox environments this depends on. tox will try to run all dependent environments before running this - environment. Format is same as :conf:`envlist` (allows factor usage). - - .. warning:: - - ``depends`` does not pull in dependencies into the run target, for example if you select ``py27,py36,coverage`` - via the ``-e`` tox will only run those three (even if ``coverage`` may specify as ``depends`` other targets too - - such as ``py27, py35, py36, py37``). - Jenkins override ++++++++++++++++ @@ -543,6 +524,25 @@ Complete list of settings that you can put into ``testenv*`` sections: the environment to the user upon listing environments for the command line with any level of verbosity higher than zero. +.. conf:: parallel_show_output ^ bool ^ false + + .. versionadded:: 3.7.0 + + If set to True the content of the output will always be shown when running in parallel mode. + +.. conf:: depends ^ comma separated values + + .. versionadded:: 3.7.0 + + tox environments this depends on. tox will try to run all dependent environments before running this + environment. Format is same as :conf:`envlist` (allows factor usage). + + .. warning:: + + ``depends`` does not pull in dependencies into the run target, for example if you select ``py27,py36,coverage`` + via the ``-e`` tox will only run those three (even if ``coverage`` may specify as ``depends`` other targets too - + such as ``py27, py35, py36, py37``). + Substitutions ------------- From f6c42eb98d7b82b9f624ca99134cbf7e6a4dd2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 8 Jul 2019 20:11:55 +0200 Subject: [PATCH 52/72] Use importlib.metadata from the standard library on Python 3.8+ (#1368) * Use importlib.metadata from the standard library on Python 3.8+ Fixes https://github.com/tox-dev/tox/issues/1367 * Azure Pipelines: Add Python 3.8 --- CONTRIBUTORS | 1 + azure-pipelines.yml | 2 ++ docs/changelog/1367.misc.rst | 2 ++ setup.cfg | 2 +- src/tox/config/__init__.py | 2 +- src/tox/session/commands/show_config.py | 2 +- src/tox/util/stdlib.py | 5 +++++ tox.ini | 2 +- 8 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/1367.misc.rst diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 39d848b4f..ad0168f64 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -58,6 +58,7 @@ Mattieu Agopian Michael Manganiello Mickaël Schoentgen Mikhail Kyshtymov +Miro Hrončok Monty Taylor Morgan Fainberg Nick Douma diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bce5e39b6..c87d0b57b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,6 +29,8 @@ jobs: jobs: fix_lint: null docs: null + py38: + image: [linux] py37: image: [linux, windows, macOs] py27: diff --git a/docs/changelog/1367.misc.rst b/docs/changelog/1367.misc.rst new file mode 100644 index 000000000..f1643f3c4 --- /dev/null +++ b/docs/changelog/1367.misc.rst @@ -0,0 +1,2 @@ +Replace ``importlib_metadata`` backport with ``importlib.metadata`` +from the standard library on Python ``3.8+`` - by :user:`hroncok` diff --git a/setup.cfg b/setup.cfg index 48dab494a..943b56b1a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ classifiers = packages = find: python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* install_requires = - importlib-metadata >= 0.12, <1 + importlib-metadata >= 0.12, <1;python_version<"3.8" packaging >= 14 pluggy >= 0.12.0, <1 py >= 1.4.17, <2 diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index c1a3239ef..849b31f7e 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -15,7 +15,6 @@ from subprocess import list2cmdline from threading import Thread -import importlib_metadata import pluggy import py import toml @@ -34,6 +33,7 @@ verbosity1, ) from tox.util.path import ensure_empty_dir +from tox.util.stdlib import importlib_metadata from .parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY from .parallel import add_parallel_config, add_parallel_flags diff --git a/src/tox/session/commands/show_config.py b/src/tox/session/commands/show_config.py index 11c5cf267..efb713ac7 100644 --- a/src/tox/session/commands/show_config.py +++ b/src/tox/session/commands/show_config.py @@ -1,13 +1,13 @@ import sys from collections import OrderedDict -import importlib_metadata from packaging.requirements import Requirement from packaging.utils import canonicalize_name from six import StringIO from six.moves import configparser from tox import reporter +from tox.util.stdlib import importlib_metadata DO_NOT_SHOW_CONFIG_ATTRIBUTES = ( "interpreters", diff --git a/src/tox/util/stdlib.py b/src/tox/util/stdlib.py index 0b2585949..5f687b737 100644 --- a/src/tox/util/stdlib.py +++ b/src/tox/util/stdlib.py @@ -3,6 +3,11 @@ from contextlib import contextmanager from tempfile import TemporaryFile +if sys.version_info >= (3, 8): + from importlib import metadata as importlib_metadata # noqa +else: + import importlib_metadata # noqa + def is_main_thread(): """returns true if we are within the main thread""" diff --git a/tox.ini b/tox.ini index fa5b321de..fcb1418cf 100644 --- a/tox.ini +++ b/tox.ini @@ -139,7 +139,7 @@ include_trailing_comma = True force_grid_wrap = 0 line_length = 99 known_first_party = tox,tests -known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,importlib_metadata,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,six,sphinx,toml +known_third_party = apiclient,docutils,filelock,flaky,freezegun,git,httplib2,oauth2client,packaging,pathlib2,pluggy,py,pytest,setuptools,six,sphinx,toml [testenv:release] description = do a release, required posarg of the version number From 4605849206528e6125a2d0878027c815fb8a6e04 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 11 Jul 2019 16:43:16 +0200 Subject: [PATCH 53/72] Make changelog instructions visible @ folder view (#1370) * Make changelog instructions visible @ folder view * Add a change fragment --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- docs/changelog/1370.misc.show-change-fragments-help.rst | 1 + docs/changelog/{examples.rst => README.rst} | 0 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1370.misc.show-change-fragments-help.rst rename docs/changelog/{examples.rst => README.rst} (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 823822f5a..4e4e0d69b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,5 +18,5 @@ much about the checklist - we will help you get started. * if PR has no issue: consider creating one first or change it to the PR number after creating the PR * "sign" fragment with "by :user:``" * please use full sentences with correct case and punctuation, for example: "Fix issue with non-ascii contents in doctest text files - by :user:`superuser`." - * also see [examples](../tree/master/docs/changelog/examples.rst) + * also see [examples](../tree/master/docs/changelog) - [ ] added yourself to `CONTRIBUTORS` (preserving alphabetical order) diff --git a/docs/changelog/1370.misc.show-change-fragments-help.rst b/docs/changelog/1370.misc.show-change-fragments-help.rst new file mode 100644 index 000000000..03af261df --- /dev/null +++ b/docs/changelog/1370.misc.show-change-fragments-help.rst @@ -0,0 +1 @@ +Render the change fragment help on the ``docs/changelog/`` directory view on GitHub — by :user:`webknjaz` diff --git a/docs/changelog/examples.rst b/docs/changelog/README.rst similarity index 100% rename from docs/changelog/examples.rst rename to docs/changelog/README.rst From 1e6430c5e08ae20a15668ed183b1fcc81ed8f050 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 12 Jul 2019 01:25:28 +0200 Subject: [PATCH 54/72] Enforce rst file extension @ change fragments (#1371) --- .github/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/config.yml b/.github/config.yml index 19c3656b1..4650149a9 100644 --- a/.github/config.yml +++ b/.github/config.yml @@ -1,2 +1,5 @@ +chronographer: + enforce_name: + suffix: .rst rtd: project: tox From 1908b198be5fd6e92143a78b43caf636965bbc6e Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Fri, 26 Jul 2019 14:07:51 -0700 Subject: [PATCH 55/72] Fix tests running under pypy (#1379) --- azure-pipelines.yml | 4 + docs/changelog/1378.bugfix.rst | 1 + src/tox/_pytestplugin.py | 2 +- tests/integration/test_package_int.py | 103 ++++++++++--------- tests/unit/config/test_config.py | 16 +-- tests/unit/interpreters/test_interpreters.py | 1 + tests/unit/test_venv.py | 4 +- tests/unit/test_z_cmdline.py | 12 ++- 8 files changed, 80 insertions(+), 63 deletions(-) create mode 100644 docs/changelog/1378.bugfix.rst diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c87d0b57b..803a210ef 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,6 +35,10 @@ jobs: image: [linux, windows, macOs] py27: image: [linux, windows, macOs] + pypy: + image: [linux] + pypy3: + image: [linux] py36: image: [linux, windows, macOs] py35: diff --git a/docs/changelog/1378.bugfix.rst b/docs/changelog/1378.bugfix.rst new file mode 100644 index 000000000..f4ac64819 --- /dev/null +++ b/docs/changelog/1378.bugfix.rst @@ -0,0 +1 @@ +Fix ``current_tox_py`` for ``pypy`` / ``pypy3`` - by :user:`asottile` diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 7637880ff..1ff89155f 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -577,7 +577,7 @@ def build_session(config): def current_tox_py(): """generate the current (test runners) python versions key e.g. py37 when running under Python 3.7""" - return "py{}".format("".join(str(i) for i in sys.version_info[0:2])) + return "{}{}{}".format("pypy" if tox.INFO.IS_PYPY else "py", *sys.version_info) def pytest_runtest_setup(item): diff --git a/tests/integration/test_package_int.py b/tests/integration/test_package_int.py index b91848520..a982c2a07 100644 --- a/tests/integration/test_package_int.py +++ b/tests/integration/test_package_int.py @@ -14,17 +14,19 @@ def test_package_setuptools(initproj, cmd): initproj( "magic-0.1", filedefs={ - "tox.ini": """ - [tox] - isolated_build = true - [testenv:.package] - basepython = python - """, - "pyproject.toml": """ - [build-system] - requires = ["setuptools >= 35.0.2", "setuptools_scm >= 2.0.0, <3"] - build-backend = "setuptools.build_meta" - """, + "tox.ini": """\ + [tox] + isolated_build = true + [testenv:.package] + basepython = {} + """.format( + sys.executable + ), + "pyproject.toml": """\ + [build-system] + requires = ["setuptools >= 35.0.2", "setuptools_scm >= 2.0.0, <3"] + build-backend = "setuptools.build_meta" + """, }, ) run(cmd, "magic-0.1.tar.gz") @@ -37,26 +39,28 @@ def test_package_flit(initproj, cmd): initproj( "magic-0.1", filedefs={ - "tox.ini": """ - [tox] - isolated_build = true - [testenv:.package] - basepython = python - """, - "pyproject.toml": """ - [build-system] - requires = ["flit"] - build-backend = "flit.buildapi" - - [tool.flit.metadata] - module = "magic" - author = "Happy Harry" - author-email = "happy@harry.com" - home-page = "https://github.com/happy-harry/is" - requires = [ - "tox", - ] - """, + "tox.ini": """\ + [tox] + isolated_build = true + [testenv:.package] + basepython = {} + """.format( + sys.executable + ), + "pyproject.toml": """\ + [build-system] + requires = ["flit"] + build-backend = "flit.buildapi" + + [tool.flit.metadata] + module = "magic" + author = "Happy Harry" + author-email = "happy@harry.com" + home-page = "https://github.com/happy-harry/is" + requires = [ + "tox", + ] + """, ".gitignore": ".tox", }, add_missing_setup_py=False, @@ -78,24 +82,25 @@ def test_package_poetry(initproj, cmd): initproj( "magic-0.1", filedefs={ - "tox.ini": """ - [tox] - isolated_build = true - [testenv:.package] - basepython = python - """, - "pyproject.toml": """ - [build-system] - requires = ["poetry>=0.12"] - build-backend = "poetry.masonry.api" - - [tool.poetry] - name = "magic" - version = "0.1.0" - description = "" - authors = ["Name "] - - """, + "tox.ini": """\ + [tox] + isolated_build = true + [testenv:.package] + basepython = {} + """.format( + sys.executable + ), + "pyproject.toml": """\ + [build-system] + requires = ["poetry>=0.12"] + build-backend = "poetry.masonry.api" + + [tool.poetry] + name = "magic" + version = "0.1.0" + description = "" + authors = ["Name "] + """, ".gitignore": ".tox", }, add_missing_setup_py=False, diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 2f019b791..7893d012b 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -1909,17 +1909,21 @@ def test_default_factors(self, newconfig): def test_default_factors_conflict(self, newconfig, capsys): with pytest.warns(UserWarning, match=r"conflicting basepython .*"): + exe = "pypy3" if tox.INFO.IS_PYPY else "python3" + env = "pypy27" if tox.INFO.IS_PYPY else "py27" config = newconfig( - """ + """\ [testenv] - basepython=python3 - [testenv:py27] + basepython={} + [testenv:{}] commands = python --version - """ + """.format( + exe, env + ) ) assert len(config.envconfigs) == 1 - envconfig = config.envconfigs["py27"] - assert envconfig.basepython == "python3" + envconfig = config.envconfigs[env] + assert envconfig.basepython == exe def test_default_factors_conflict_lying_name( self, newconfig, capsys, tmpdir, recwarn, monkeypatch diff --git a/tests/unit/interpreters/test_interpreters.py b/tests/unit/interpreters/test_interpreters.py index ea6f65dff..cc2c3f58a 100644 --- a/tests/unit/interpreters/test_interpreters.py +++ b/tests/unit/interpreters/test_interpreters.py @@ -28,6 +28,7 @@ def create_interpreters_instance(): return Interpreters(hook=pm.hook) +@pytest.mark.skipif(tox.INFO.IS_PYPY, reason="testing cpython interpreter discovery") def test_tox_get_python_executable(): class envconfig: basepython = sys.executable diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 05231121f..0e302709a 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -446,8 +446,8 @@ def test_install_command_not_installed_bash(newmocksession): def test_install_python3(newmocksession): - if not py.path.local.sysfind("python3"): - pytest.skip("needs python3") + if not py.path.local.sysfind("python3") or tox.INFO.IS_PYPY: + pytest.skip("needs cpython3") mocksession = newmocksession( [], """\ diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index ae74648aa..20e5d0ce5 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -784,11 +784,13 @@ def test_notest(initproj, cmd): initproj( "example123", filedefs={ - "tox.ini": """ - # content of: tox.ini - [testenv:py26] - basepython=python - """ + "tox.ini": """\ + # content of: tox.ini + [testenv:py26] + basepython={} + """.format( + sys.executable + ) }, ) result = cmd("-v", "--notest") From 8aae84d0ce9f034af980f5cfbee88c28640a85bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 26 Jul 2019 22:09:01 +0100 Subject: [PATCH 56/72] Fix CI (#1364) --- tests/unit/interpreters/test_interpreters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/interpreters/test_interpreters.py b/tests/unit/interpreters/test_interpreters.py index cc2c3f58a..c713015fe 100644 --- a/tests/unit/interpreters/test_interpreters.py +++ b/tests/unit/interpreters/test_interpreters.py @@ -58,10 +58,10 @@ def assert_version_in_output(exe, version): continue exe = get_exe(name) assert_version_in_output(exe, "{}.{}".format(major, minor)) - + has_py_exe = py.path.local.sysfind("py") is not None for major in (2, 3): name = "python{}".format(major) - if tox.INFO.IS_WIN: + if has_py_exe: error_code = subprocess.call(("py", "-{}".format(major), "-c", "")) if error_code: continue From 6b19ead56ec7df1d60ec4f6255f14acef05bee38 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 27 Jul 2019 09:19:23 -0700 Subject: [PATCH 57/72] Work around broken pyparsing release (#1380) --- tests/integration/test_provision_int.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_provision_int.py b/tests/integration/test_provision_int.py index 6a8ac1877..01f3cfec8 100644 --- a/tests/integration/test_provision_int.py +++ b/tests/integration/test_provision_int.py @@ -17,14 +17,16 @@ def test_provision_missing(initproj, cmd): initproj( "pkg123-0.7", filedefs={ - "tox.ini": """ - [tox] - skipsdist=True - minversion = 3.7.0 - requires = setuptools == 40.6.3 - [testenv] - commands=python -c "import sys; print(sys.executable); raise SystemExit(1)" - """ + "tox.ini": """\ + [tox] + skipsdist=True + minversion = 3.7.0 + requires = + setuptools == 40.6.3 + pyparsing!=2.4.1,!=2.4.1.1;python_version=="3.4" + [testenv] + commands=python -c "import sys; print(sys.executable); raise SystemExit(1)" + """ }, ) result = cmd("-e", "py") From d0e255c8d3d4d643614596d4a859d443a95c27c5 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 27 Jul 2019 09:20:02 -0700 Subject: [PATCH 58/72] Un-hardcode the supported factors again (#1377) --- docs/changelog/1374.feature.rst | 1 + docs/changelog/1377.fix.rst | 1 + src/tox/config/__init__.py | 48 +++++++++++--------- src/tox/constants.py | 31 ++----------- tests/unit/config/test_config.py | 34 +++++++------- tests/unit/interpreters/test_interpreters.py | 2 +- tests/unit/test_z_cmdline.py | 25 ++++++---- 7 files changed, 63 insertions(+), 79 deletions(-) create mode 100644 docs/changelog/1374.feature.rst create mode 100644 docs/changelog/1377.fix.rst diff --git a/docs/changelog/1374.feature.rst b/docs/changelog/1374.feature.rst new file mode 100644 index 000000000..171911f34 --- /dev/null +++ b/docs/changelog/1374.feature.rst @@ -0,0 +1 @@ +Add support for minor versions with multiple digits ``tox -e py310` works for ``python3.10`` - by :user:`asottile` diff --git a/docs/changelog/1377.fix.rst b/docs/changelog/1377.fix.rst new file mode 100644 index 000000000..2834b85b0 --- /dev/null +++ b/docs/changelog/1377.fix.rst @@ -0,0 +1 @@ +Fix regression failing to detect future and past ``py##`` factors - by :user:`asottile` diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index 849b31f7e..b5df35833 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -51,9 +51,6 @@ Import hookimpl directly from tox instead. """ -default_factors = tox.PYTHON.DEFAULT_FACTORS -"""DEPRECATED MOVE - please update to new location.""" - WITHIN_PROVISION = os.environ.get(str("TOX_PROVISION")) == "1" @@ -549,11 +546,21 @@ def basepython_default(testenv_config, value): python conflict is set in which case the factor name implied version if forced """ for factor in testenv_config.factors: - if factor in tox.PYTHON.DEFAULT_FACTORS: - implied_python = tox.PYTHON.DEFAULT_FACTORS[factor] + match = tox.PYTHON.PY_FACTORS_RE.match(factor) + if match: + base_exe = {"py": "python"}.get(match.group(1), match.group(1)) + version_s = match.group(2) + if not version_s: + version_info = () + elif len(version_s) == 1: + version_info = (version_s,) + else: + version_info = (version_s[0], version_s[1:]) + implied_version = ".".join(version_info) + implied_python = "{}{}".format(base_exe, implied_version) break else: - implied_python, factor = None, None + implied_python, version_info, implied_version = None, (), "" if testenv_config.config.ignore_basepython_conflict and implied_python is not None: return implied_python @@ -561,23 +568,20 @@ def basepython_default(testenv_config, value): proposed_python = (implied_python or sys.executable) if value is None else str(value) if implied_python is not None and implied_python != proposed_python: testenv_config.basepython = proposed_python - match = tox.PYTHON.PY_FACTORS_RE.match(factor) - implied_version = match.group(2) if match else None - if implied_version is not None: - python_info_for_proposed = testenv_config.python_info - if not isinstance(python_info_for_proposed, NoInterpreterInfo): - proposed_version = "".join( - str(i) for i in python_info_for_proposed.version_info[0:2] - ) - # '27'.startswith('2') or '27'.startswith('27') - if not proposed_version.startswith(implied_version): - # TODO(stephenfin): Raise an exception here in tox 4.0 - warnings.warn( - "conflicting basepython version (set {}, should be {}) for env '{}';" - "resolve conflict or set ignore_basepython_conflict".format( - proposed_version, implied_version, testenv_config.envname - ) + python_info_for_proposed = testenv_config.python_info + if not isinstance(python_info_for_proposed, NoInterpreterInfo): + proposed_version = ".".join( + str(x) for x in python_info_for_proposed.version_info[: len(version_info)] + ) + if proposed_version != implied_version: + # TODO(stephenfin): Raise an exception here in tox 4.0 + warnings.warn( + "conflicting basepython version (set {}, should be {}) for env '{}';" + "resolve conflict or set ignore_basepython_conflict".format( + proposed_version, implied_version, testenv_config.envname ) + ) + return proposed_python parser.add_testenv_attribute( diff --git a/src/tox/constants.py b/src/tox/constants.py index 8dcc86bf8..c31f2602f 100644 --- a/src/tox/constants.py +++ b/src/tox/constants.py @@ -9,36 +9,11 @@ _THIS_FILE = os.path.realpath(os.path.abspath(__file__)) -def _construct_default_factors(cpython_versions, pypy_versions, other_interpreters): - default_factors = {"py": sys.executable, "py2": "python2", "py3": "python3"} - default_factors.update( - { - "py{}{}".format(major, minor): "python{}.{}".format(major, minor) - for major, minor in cpython_versions - } - ) - default_factors.update({exc: exc for exc in ["pypy", "pypy2", "pypy3"]}) - default_factors.update( - { - "pypy{}{}".format(major, minor): "pypy{}.{}".format(major, minor) - for major, minor in pypy_versions - } - ) - default_factors.update({interpreter: interpreter for interpreter in other_interpreters}) - return default_factors - - class PYTHON: - PY_FACTORS_RE = re.compile("^(?!py$)(py|pypy|jython)([2-9][0-9]?)?$") - CPYTHON_VERSION_TUPLES = [(2, 7), (3, 4), (3, 5), (3, 6), (3, 7), (3, 8)] - PYPY_VERSION_TUPLES = [(2, 7), (3, 5)] - OTHER_PYTHON_INTERPRETERS = ["jython"] - DEFAULT_FACTORS = _construct_default_factors( - CPYTHON_VERSION_TUPLES, PYPY_VERSION_TUPLES, OTHER_PYTHON_INTERPRETERS - ) - CURRENT_RELEASE_ENV = "py36" + PY_FACTORS_RE = re.compile("^(?!py$)(py|pypy|jython)([2-9][0-9]?[0-9]?)?$") + CURRENT_RELEASE_ENV = "py37" """Should hold currently released py -> for easy updating""" - QUICKSTART_PY_ENVS = ["py27", "py34", "py35", CURRENT_RELEASE_ENV, "pypy", "jython"] + QUICKSTART_PY_ENVS = ["py27", "py35", "py36", CURRENT_RELEASE_ENV, "pypy", "jython"] """For choices in tox-quickstart""" diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 7893d012b..d3b17970f 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -2194,26 +2194,24 @@ def test_no_implicit_venv_from_cli_with_envlist(self, newconfig): assert "typo-factor" not in config.envconfigs def test_correct_basepython_chosen_from_default_factors(self, newconfig): - envlist = list(tox.PYTHON.DEFAULT_FACTORS.keys()) - config = newconfig([], "[tox]\nenvlist={}".format(", ".join(envlist))) - assert config.envlist == envlist + envs = { + "py": sys.executable, + "py2": "python2", + "py3": "python3", + "py27": "python2.7", + "py36": "python3.6", + "py310": "python3.10", + "pypy": "pypy", + "pypy2": "pypy2", + "pypy3": "pypy3", + "pypy36": "pypy3.6", + "jython": "jython", + } + config = newconfig([], "[tox]\nenvlist={}".format(", ".join(envs))) + assert set(config.envlist) == set(envs) for name in config.envlist: basepython = config.envconfigs[name].basepython - if name == "jython": - assert basepython == "jython" - elif name in ("pypy2", "pypy3"): - assert basepython == "pypy" + name[-1] - elif name in ("py2", "py3"): - assert basepython == "python" + name[-1] - elif name == "pypy": - assert basepython == name - elif name == "py": - assert "python" in basepython or "pypy" in basepython - elif "pypy" in name: - assert basepython == "pypy{}.{}".format(name[-2], name[-1]) - else: - assert name.startswith("py") - assert basepython == "python{}.{}".format(name[2], name[3]) + assert basepython == envs[name] def test_envlist_expansion(self, newconfig): inisource = """ diff --git a/tests/unit/interpreters/test_interpreters.py b/tests/unit/interpreters/test_interpreters.py index c713015fe..31d67ac7e 100644 --- a/tests/unit/interpreters/test_interpreters.py +++ b/tests/unit/interpreters/test_interpreters.py @@ -46,7 +46,7 @@ def assert_version_in_output(exe, version): p = tox_get_python_executable(envconfig) assert p == py.path.local(sys.executable) - for major, minor in tox.PYTHON.CPYTHON_VERSION_TUPLES: + for major, minor in [(2, 7), (3, 5), (3, 6), (3, 7), (3, 8)]: name = "python{}.{}".format(major, minor) if tox.INFO.IS_WIN: pydir = "python{}{}".format(major, minor) diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index 20e5d0ce5..f8681f74a 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -164,26 +164,31 @@ def test_unknown_interpreter_and_env(cmd, initproj): "interp123-0.5", filedefs={ "tests": {"test_hello.py": "def test_hello(): pass"}, - "tox.ini": """ - [testenv:python] - basepython=xyz_unknown_interpreter - [testenv] - changedir=tests - skip_install = true - """, + "tox.ini": """\ + [testenv:python] + basepython=xyz_unknown_interpreter + [testenv] + changedir=tests + skip_install = true + """, }, ) result = cmd() result.assert_fail() - assert any( - "ERROR: InterpreterNotFound: xyz_unknown_interpreter" == l for l in result.outlines - ), result.outlines + assert "ERROR: InterpreterNotFound: xyz_unknown_interpreter" in result.outlines result = cmd("-exyz") result.assert_fail() assert result.out == "ERROR: unknown environment 'xyz'\n" +def test_unknown_interpreter_factor(cmd, initproj): + initproj("py21", filedefs={"tox.ini": "[testenv]\nskip_install=true"}) + result = cmd("-e", "py21") + result.assert_fail() + assert "ERROR: InterpreterNotFound: python2.1" in result.outlines + + def test_unknown_interpreter(cmd, initproj): initproj( "interp123-0.5", From 57f8cc2cb34e6bacc5f6a5e05334a2a2c4fa7cc1 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 27 Jul 2019 13:53:43 -0700 Subject: [PATCH 59/72] Fix testsuite and honor environment markers in requires (#1382) --- docs/changelog/1380.bugfix.rst | 1 + src/tox/config/__init__.py | 3 ++ tests/integration/test_provision_int.py | 3 +- tests/unit/session/test_provision.py | 37 +++++++++++++++++-------- 4 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 docs/changelog/1380.bugfix.rst diff --git a/docs/changelog/1380.bugfix.rst b/docs/changelog/1380.bugfix.rst new file mode 100644 index 000000000..c7fdadf16 --- /dev/null +++ b/docs/changelog/1380.bugfix.rst @@ -0,0 +1 @@ +Honor environment markers in ``requires`` list - by :user:`asottile` diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index b5df35833..1d66ed7ae 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -1165,6 +1165,9 @@ def ensure_requires_satisfied(config, requires, min_version): # noinspection PyBroadException try: package = requirements.Requirement(require) + # check if the package even applies + if package.marker and not package.marker.evaluate({"extra": ""}): + continue package_name = canonicalize_name(package.name) if package_name not in exists: deps.append(DepConfig(require, None)) diff --git a/tests/integration/test_provision_int.py b/tests/integration/test_provision_int.py index 01f3cfec8..b002f0ee7 100644 --- a/tests/integration/test_provision_int.py +++ b/tests/integration/test_provision_int.py @@ -23,7 +23,8 @@ def test_provision_missing(initproj, cmd): minversion = 3.7.0 requires = setuptools == 40.6.3 - pyparsing!=2.4.1,!=2.4.1.1;python_version=="3.4" + # remove when 2.4.2 is released or python3.4 is dropped + pyparsing!=2.4.1,!=2.4.1.1 [testenv] commands=python -c "import sys; print(sys.executable); raise SystemExit(1)" """ diff --git a/tests/unit/session/test_provision.py b/tests/unit/session/test_provision.py index e8f7abc23..c2840197c 100644 --- a/tests/unit/session/test_provision.py +++ b/tests/unit/session/test_provision.py @@ -24,10 +24,10 @@ def test_provision_min_version_is_requires(newconfig, next_tox_major): with pytest.raises(MissingRequirement) as context: newconfig( [], - """ + """\ [tox] minversion = {} - """.format( + """.format( next_tox_major ), ) @@ -45,10 +45,10 @@ def test_provision_min_version_is_requires(newconfig, next_tox_major): def test_provision_tox_change_name(newconfig): config = newconfig( [], - """ + """\ [tox] provision_tox_env = magic - """, + """, ) assert config.provision_tox_env == "magic" @@ -58,12 +58,12 @@ def test_provision_basepython_global_only(newconfig, next_tox_major): with pytest.raises(MissingRequirement) as context: newconfig( [], - """ + """\ [tox] minversion = {} [testenv] basepython = what - """.format( + """.format( next_tox_major ), ) @@ -77,12 +77,12 @@ def test_provision_basepython_local(newconfig, next_tox_major): with pytest.raises(MissingRequirement) as context: newconfig( [], - """ + """\ [tox] minversion = {} [testenv:.tox] basepython = what - """.format( + """.format( next_tox_major ), ) @@ -95,10 +95,10 @@ def test_provision_bad_requires(newconfig, capsys, monkeypatch): with pytest.raises(BadRequirement): newconfig( [], - """ + """\ [tox] requires = sad >sds d ok - """, + """, ) out, err = capsys.readouterr() assert "ERROR: failed to parse InvalidRequirement" in out @@ -208,7 +208,7 @@ def test_provision_non_canonical_dep( initproj( "w-0.1", { - "tox.ini": """ + "tox.ini": """\ [tox] envlist = py requires = @@ -231,6 +231,21 @@ def test_provision_non_canonical_dep( result.assert_success(is_run_test_env=False) +def test_provision_requirement_with_environment_marker(cmd, initproj): + initproj( + "proj", + { + "tox.ini": """\ + [tox] + requires = + package-that-does-not-exist;python_version=="1.0" + """ + }, + ) + result = cmd("-e", "py", "-vv") + result.assert_success(is_run_test_env=False) + + def space_path2url(path): at_path = str(path) if " " not in at_path: From d056fd2c52b551b5bb8df2cdb201086f267b8264 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Mon, 29 Jul 2019 11:40:17 -0700 Subject: [PATCH 60/72] Improve recreate check by looking for .tox-config1 (#1383) --- docs/changelog/1383.bugfix.rst | 1 + src/tox/venv.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1383.bugfix.rst diff --git a/docs/changelog/1383.bugfix.rst b/docs/changelog/1383.bugfix.rst new file mode 100644 index 000000000..3445944ba --- /dev/null +++ b/docs/changelog/1383.bugfix.rst @@ -0,0 +1 @@ +improve recreate check by allowing directories containing ``.tox-config1`` (the marker file created by tox) - by :user:`asottile` diff --git a/src/tox/venv.py b/src/tox/venv.py index 607f8a84a..04fc14617 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -698,7 +698,7 @@ def cleanup_for_venv(venv): # an error if venv.path.exists(): dir_items = set(os.listdir(str(venv.path))) - {".lock", "log"} - dir_items = {p for p in dir_items if not p.startswith(".tox-")} + dir_items = {p for p in dir_items if not p.startswith(".tox-") or p == ".tox-config1"} else: dir_items = set() @@ -707,6 +707,8 @@ def cleanup_for_venv(venv): not venv.path.exists() # does exist, but it's empty => OK or not dir_items + # tox has marked this as an environment it has created in the past + or ".tox-config1" in dir_items # it exists and we're on windows with Lib and Scripts => OK or (INFO.IS_WIN and dir_items > {"Scripts", "Lib"}) # non-windows, with lib and bin => OK From 94436c320aa06a323cfc6bc5e2c2277a0586f2ab Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Wed, 31 Jul 2019 01:28:25 -0700 Subject: [PATCH 61/72] Remove dependence on md5 (#1385) --- docs/changelog/1384.feature.rst | 1 + docs/example/result.rst | 3 +-- src/tox/_pytestplugin.py | 4 ++-- src/tox/logs/env.py | 1 - src/tox/venv.py | 26 +++++++++++++------------- tests/unit/test_result.py | 6 +----- tests/unit/test_venv.py | 4 ++-- 7 files changed, 20 insertions(+), 25 deletions(-) create mode 100644 docs/changelog/1384.feature.rst diff --git a/docs/changelog/1384.feature.rst b/docs/changelog/1384.feature.rst new file mode 100644 index 000000000..ab6146ef4 --- /dev/null +++ b/docs/changelog/1384.feature.rst @@ -0,0 +1 @@ +Remove dependence on ``md5`` hashing algorithm - by :user:`asottile` diff --git a/docs/example/result.rst b/docs/example/result.rst index 723363936..cdb3cfb2e 100644 --- a/docs/example/result.rst +++ b/docs/example/result.rst @@ -40,8 +40,7 @@ This will create a json-formatted result file using this schema: "platform": "linux2", "installpkg": { "basename": "tox-1.6.0.dev1.zip", - "sha256": "b6982dde5789a167c4c35af0d34ef72176d0575955f5331ad04aee9f23af4326", - "md5": "27ead99fd7fa39ee7614cede6bf175a6" + "sha256": "b6982dde5789a167c4c35af0d34ef72176d0575955f5331ad04aee9f23af4326" }, "toxversion": "1.6.0.dev1", "reportversion": "1" diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 1ff89155f..5d34258e3 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -503,7 +503,7 @@ class ProxyCurrentPython: def readconfig(cls, path): if path.dirname.endswith("{}py".format(os.sep)): return CreationConfig( - base_resolved_python_md5=getdigest(sys.executable), + base_resolved_python_sha256=getdigest(sys.executable), base_resolved_python_path=sys.executable, tox_version=tox.__version__, sitepackages=False, @@ -513,7 +513,7 @@ def readconfig(cls, path): ) elif path.dirname.endswith("{}.package".format(os.sep)): return CreationConfig( - base_resolved_python_md5=getdigest(sys.executable), + base_resolved_python_sha256=getdigest(sys.executable), base_resolved_python_path=sys.executable, tox_version=tox.__version__, sitepackages=False, diff --git a/src/tox/logs/env.py b/src/tox/logs/env.py index f134440a7..d83753b71 100644 --- a/src/tox/logs/env.py +++ b/src/tox/logs/env.py @@ -36,7 +36,6 @@ def set_header(self, installpkg): :param py.path.local installpkg: Path ot the package. """ self.dict["installpkg"] = { - "md5": installpkg.computehash("md5"), "sha256": installpkg.computehash("sha256"), "basename": installpkg.basename, } diff --git a/src/tox/venv.py b/src/tox/venv.py index 04fc14617..ef1deb772 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -23,7 +23,7 @@ class CreationConfig: def __init__( self, - base_resolved_python_md5, + base_resolved_python_sha256, base_resolved_python_path, tox_version, sitepackages, @@ -31,7 +31,7 @@ def __init__( deps, alwayscopy, ): - self.base_resolved_python_md5 = base_resolved_python_md5 + self.base_resolved_python_sha256 = base_resolved_python_sha256 self.base_resolved_python_path = base_resolved_python_path self.tox_version = tox_version self.sitepackages = sitepackages @@ -41,7 +41,7 @@ def __init__( def writeconfig(self, path): lines = [ - "{} {}".format(self.base_resolved_python_md5, self.base_resolved_python_path), + "{} {}".format(self.base_resolved_python_sha256, self.base_resolved_python_path), "{} {:d} {:d} {:d}".format( self.tox_version, self.sitepackages, self.usedevelop, self.alwayscopy ), @@ -64,11 +64,11 @@ def readconfig(cls, path): alwayscopy = bool(int(alwayscopy)) deps = [] for line in lines: - base_resolved_python_md5, depstring = line.split(None, 1) - deps.append((base_resolved_python_md5, depstring)) - base_resolved_python_md5, base_resolved_python_path = base_resolved_python_info + base_resolved_python_sha256, depstring = line.split(None, 1) + deps.append((base_resolved_python_sha256, depstring)) + base_resolved_python_sha256, base_resolved_python_path = base_resolved_python_info return CreationConfig( - base_resolved_python_md5, + base_resolved_python_sha256, base_resolved_python_path, tox_version, sitepackages, @@ -81,7 +81,7 @@ def readconfig(cls, path): def matches_with_reason(self, other, deps_matches_subset=False): for attr in ( - "base_resolved_python_md5", + "base_resolved_python_sha256", "base_resolved_python_path", "tox_version", "sitepackages", @@ -266,11 +266,11 @@ def _getliveconfig(self): alwayscopy = self.envconfig.alwayscopy deps = [] for dep in self.get_resolved_dependencies(): - dep_name_md5 = getdigest(dep.name) - deps.append((dep_name_md5, dep.name)) - base_resolved_python_md5 = getdigest(base_resolved_python_path) + dep_name_sha256 = getdigest(dep.name) + deps.append((dep_name_sha256, dep.name)) + base_resolved_python_sha256 = getdigest(base_resolved_python_path) return CreationConfig( - base_resolved_python_md5, + base_resolved_python_sha256, base_resolved_python_path, version, sitepackages, @@ -629,7 +629,7 @@ def getdigest(path): path = py.path.local(path) if not path.check(file=1): return "0" * 32 - return path.computehash() + return path.computehash("sha256") def prepend_shebang_interpreter(args): diff --git a/tests/unit/test_result.py b/tests/unit/test_result.py index 23e108606..04da6b6b4 100644 --- a/tests/unit/test_result.py +++ b/tests/unit/test_result.py @@ -38,11 +38,7 @@ def test_set_header(pkg): assert replog.dict["toxversion"] == tox.__version__ assert replog.dict["platform"] == sys.platform assert replog.dict["host"] == socket.getfqdn() - expected = { - "basename": "hello-1.0.tar.gz", - "md5": pkg.computehash("md5"), - "sha256": pkg.computehash("sha256"), - } + expected = {"basename": "hello-1.0.tar.gz", "sha256": pkg.computehash("sha256")} env_log = replog.get_envlog("a") env_log.set_header(installpkg=pkg) assert env_log.dict["installpkg"] == expected diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 0e302709a..a574326ad 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -548,9 +548,9 @@ def test_matchingdependencies_latest(self, newconfig, mocksession): mocksession.new_config(config) venv = mocksession.getvenv("python") cconfig = venv._getliveconfig() - md5, path = cconfig.deps[0] + sha256, path = cconfig.deps[0] assert path == xyz2 - assert md5 == path.computehash() + assert sha256 == path.computehash("sha256") def test_python_recreation(self, tmpdir, newconfig, mocksession): pkg = tmpdir.ensure("package.tar.gz") From 1b8c98b9b04c334986dcfa20aeae7d38f09574d9 Mon Sep 17 00:00:00 2001 From: Stig Brautaset Date: Wed, 7 Aug 2019 23:57:33 +0100 Subject: [PATCH 62/72] Fix typo (#1392) --- docs/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.rst b/docs/config.rst index 3e24c5cf4..f5d856c0f 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -336,7 +336,7 @@ Complete list of settings that you can put into ``testenv*`` sections: .. versionchanged:: 2.3 - Support for index servers is now deprecated, and it's usage discouraged. + Support for index servers is now deprecated, and its usage discouraged. .. versionchanged:: 3.9 From f34764a41268a7074fda52fcb38b8f13d792a6b1 Mon Sep 17 00:00:00 2001 From: bu-tegan <46607894+bu-tegan@users.noreply.github.com> Date: Thu, 8 Aug 2019 08:55:17 -0400 Subject: [PATCH 63/72] Add quote marks so readers are not tripped up by spaces in their path names (#1393) --- docs/example/pytest.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/example/pytest.rst b/docs/example/pytest.rst index 28e4c61a8..11bd046db 100644 --- a/docs/example/pytest.rst +++ b/docs/example/pytest.rst @@ -53,7 +53,7 @@ and the following ``tox.ini`` content: changedir = tests deps = pytest # change pytest tempdir and add posargs from command line - commands = pytest --basetemp={envtmpdir} {posargs} + commands = pytest --basetemp="{envtmpdir}" {posargs} you can invoke ``tox`` in the directory where your ``tox.ini`` resides. Differently than in the previous example the ``pytest`` command @@ -75,7 +75,7 @@ to make ``tox`` use this feature: deps = pytest-xdist changedir = tests # use three sub processes - commands = pytest --basetemp={envtmpdir} \ + commands = pytest --basetemp="{envtmpdir}" \ --confcutdir=.. \ -n 3 \ {posargs} From 0bd51980e18d4e73d1e067710cd49ab64b45c8fb Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sat, 10 Aug 2019 13:08:10 -0700 Subject: [PATCH 64/72] =?UTF-8?q?Update=20URL:=20python/black=20=E2=86=92?= =?UTF-8?q?=20psf/black=20(#1395)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .pre-commit-config.yaml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386e655ba..292a75701 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: -- repo: https://github.com/python/black +- repo: https://github.com/psf/black rev: 19.3b0 hooks: - id: black diff --git a/README.md b/README.md index 224f713e0..de4c4b125 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ status](https://dev.azure.com/toxdev/tox/_apis/build/status/tox%20ci?branchName= [![Documentation status](https://readthedocs.org/projects/tox/badge/?version=latest&style=flat-square)](https://tox.readthedocs.io/en/latest/?badge=latest) [![Code style: -black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) +black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) Date: Tue, 13 Aug 2019 17:25:03 -0700 Subject: [PATCH 65/72] Update 1374.feature.rst --- docs/changelog/1374.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/1374.feature.rst b/docs/changelog/1374.feature.rst index 171911f34..76707229a 100644 --- a/docs/changelog/1374.feature.rst +++ b/docs/changelog/1374.feature.rst @@ -1 +1 @@ -Add support for minor versions with multiple digits ``tox -e py310` works for ``python3.10`` - by :user:`asottile` +Add support for minor versions with multiple digits ``tox -e py310`` works for ``python3.10`` - by :user:`asottile` From 3725c93d226e339efac106c0f4efa217cf6a363a Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Wed, 14 Aug 2019 01:23:05 -0700 Subject: [PATCH 66/72] Fix PythonSpec detection of python3.10 (#1398) --- docs/changelog/1374.bugfix.rst | 1 + docs/changelog/{1377.fix.rst => 1377.bugfix.rst} | 0 src/tox/interpreters/py_spec.py | 2 +- tests/unit/interpreters/test_py_spec.py | 6 ++++++ 4 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/1374.bugfix.rst rename docs/changelog/{1377.fix.rst => 1377.bugfix.rst} (100%) create mode 100644 tests/unit/interpreters/test_py_spec.py diff --git a/docs/changelog/1374.bugfix.rst b/docs/changelog/1374.bugfix.rst new file mode 100644 index 000000000..4c052109f --- /dev/null +++ b/docs/changelog/1374.bugfix.rst @@ -0,0 +1 @@ +Fix ``PythonSpec`` detection of ``python3.10`` - by :user:`asottile` diff --git a/docs/changelog/1377.fix.rst b/docs/changelog/1377.bugfix.rst similarity index 100% rename from docs/changelog/1377.fix.rst rename to docs/changelog/1377.bugfix.rst diff --git a/src/tox/interpreters/py_spec.py b/src/tox/interpreters/py_spec.py index 104af4177..04c062e63 100644 --- a/src/tox/interpreters/py_spec.py +++ b/src/tox/interpreters/py_spec.py @@ -53,7 +53,7 @@ def from_name(cls, base_python): if os.path.isabs(base_python): path = base_python else: - match = re.match(r"(python|pypy|jython)(\d)?(?:\.(\d))?(-(32|64))?", base_python) + match = re.match(r"(python|pypy|jython)(\d)?(?:\.(\d+))?(-(32|64))?", base_python) if match: groups = match.groups() name = groups[0] diff --git a/tests/unit/interpreters/test_py_spec.py b/tests/unit/interpreters/test_py_spec.py new file mode 100644 index 000000000..48024afde --- /dev/null +++ b/tests/unit/interpreters/test_py_spec.py @@ -0,0 +1,6 @@ +from tox.interpreters.py_spec import PythonSpec + + +def test_py_3_10(): + spec = PythonSpec.from_name("python3.10") + assert (spec.major, spec.minor) == (3, 10) From 2e2855a161c26fbbf6f14787b7fabf1bbb118c25 Mon Sep 17 00:00:00 2001 From: PJCampi Date: Sun, 18 Aug 2019 20:37:58 +0200 Subject: [PATCH 67/72] clarify behaviour if recreate is set to false (#1372) (#1399) * clarify behaviour if recreate is set to false in config.rst (#1372) * added file to changelog (#1372) * add new line at the end of changelog entry (#1372) --- CONTRIBUTORS | 1 + docs/changelog/1399.doc.rst | 1 + docs/config.rst | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 docs/changelog/1399.doc.rst diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ad0168f64..eb6112386 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -66,6 +66,7 @@ Nick Prendergast Oliver Bestwalter Paweł Adamczak Philip Thiem +Pierre-Jean Campigotto Pierre-Luc Tessier Gagné Ronald Evers Ronny Pfannschmidt diff --git a/docs/changelog/1399.doc.rst b/docs/changelog/1399.doc.rst new file mode 100644 index 000000000..dbb83cdeb --- /dev/null +++ b/docs/changelog/1399.doc.rst @@ -0,0 +1 @@ +clarify behaviour if recreate is set to false - by :user:`PJCampi` diff --git a/docs/config.rst b/docs/config.rst index f5d856c0f..728bcb4fe 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -406,6 +406,8 @@ Complete list of settings that you can put into ``testenv*`` sections: .. conf:: recreate ^ true|false ^ false Always recreate virtual environment if this option is true. + If this option is false, ``tox``'s resolution mechanism will be used to + determine whether to recreate the environment. .. conf:: downloadcache ^ PATH From 31b93fa401c989957c64284dac2075e01f07e381 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Tue, 20 Aug 2019 17:21:35 -0700 Subject: [PATCH 68/72] Remove pyparsing workaround (#1402) --- tests/integration/test_provision_int.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_provision_int.py b/tests/integration/test_provision_int.py index b002f0ee7..be890097c 100644 --- a/tests/integration/test_provision_int.py +++ b/tests/integration/test_provision_int.py @@ -23,8 +23,6 @@ def test_provision_missing(initproj, cmd): minversion = 3.7.0 requires = setuptools == 40.6.3 - # remove when 2.4.2 is released or python3.4 is dropped - pyparsing!=2.4.1,!=2.4.1.1 [testenv] commands=python -c "import sys; print(sys.executable); raise SystemExit(1)" """ From 0d0e7ddcb04885c80d4151498e2b5e5fd2ca09b5 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 24 Aug 2019 17:34:43 -0700 Subject: [PATCH 69/72] Use `not PY2` instead of `PY3` for python4.x compat (#1404) --- src/tox/_pytestplugin.py | 4 ++-- src/tox/interpreters/py_spec.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 5d34258e3..e03812995 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -210,7 +210,7 @@ def __init__(self): def clear(self): self._index = -1 - if six.PY3: + if not six.PY2: self.instance.reported_lines.clear() else: del self.instance.reported_lines[:] @@ -291,7 +291,7 @@ def __init__(self, config): self.report = ReportExpectMock() def _clearmocks(self): - if six.PY3: + if not six.PY2: self._pcalls.clear() else: del self._pcalls[:] diff --git a/src/tox/interpreters/py_spec.py b/src/tox/interpreters/py_spec.py index 04c062e63..2df06672e 100644 --- a/src/tox/interpreters/py_spec.py +++ b/src/tox/interpreters/py_spec.py @@ -26,7 +26,7 @@ def __repr__(self): def __str__(self): msg = repr(self) - return msg if six.PY3 else msg.encode("utf-8") + return msg.encode("utf-8") if six.PY2 else msg def satisfies(self, req): if req.is_abs and self.is_abs and self.path != req.path: From f39983c9459ea23ecbe3810be2848cfc4e38bf5c Mon Sep 17 00:00:00 2001 From: Faidon Liambotis Date: Fri, 30 Aug 2019 18:18:26 +0300 Subject: [PATCH 70/72] Fix various spelling mistakes across the tree (#1408) As identified by Debian's lintian: - comand -> command - suceeds -> succeeds - intepreter -> interpreter - allow(s) to -> allow(s) you/one to --- docs/changelog.rst | 18 +++++++++--------- docs/drafts/extend-envs-and-packagebuilds.md | 2 +- docs/example/general.rst | 2 +- docs/example/jenkins.rst | 2 +- docs/example/nose.rst | 2 +- docs/example/unittest.rst | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 499efce69..6446321c4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -476,7 +476,7 @@ Features ^^^^^^^^ - add ``commands_pre`` and ``commands_post`` that run before and after running - the ``commands`` (setup runs always, commands only if setup suceeds, teardown always - all + the ``commands`` (setup runs always, commands only if setup succeeds, teardown always - all run until the first failing command) - by :user:`gaborbernat` (`#167 `_) - ``pyproject.toml`` config support initially by just inline the tox.ini under ``tool.tox.legacy_tox_ini`` key; config source priority order is ``pyproject.toml``, ``tox.ini`` and then ``setup.cfg`` - by :user:`gaborbernat` (`#814 `_) - use the os environment variable ``TOX_SKIP_ENV`` to filter out tox environment names from the run list (set by ``envlist``) - by :user:`gaborbernat` (`#824 `_) @@ -543,7 +543,7 @@ Features - Switch pip invocations to use the module ``-m pip`` instead of direct invocation. This could help avoid some of the shebang limitations. - by :user:`gaborbernat` (`#935 `_) - Ability to specify package requirements for the tox run via the ``tox.ini`` (``tox`` section under key ``requires`` - PEP-508 style): can be used to specify both plugin requirements or build dependencies. - by :user:`gaborbernat` (`#783 `_) -- Allow to run multiple tox instances in parallel by providing the +- Allow one to run multiple tox instances in parallel by providing the ``--parallel--safe-build`` flag. - by :user:`gaborbernat` (`#849 `_) @@ -616,7 +616,7 @@ Features ^^^^^^^^ - Add support for multiple PyPy versions using default factors. This allows you - to use, for example, ``pypy27`` knowing that the correct intepreter will be + to use, for example, ``pypy27`` knowing that the correct interpreter will be used by default - by :user:`stephenfin` (`#19 `_) - Add support to explicitly invoke interpreter directives for environments with long path lengths. In the event that ``tox`` cannot invoke scripts with a @@ -1106,7 +1106,7 @@ v2.1.0 (2015-06-19) hackily implemented (if people want home-directory isolation we should figure out a better way to do it, possibly through a plugin) -- fix `#259 `_: passenv is now a line-list which allows to intersperse +- fix `#259 `_: passenv is now a line-list which allows interspersing comments. Thanks stefano-m. - allow envlist to be a multi-line list, to intersperse comments @@ -1148,7 +1148,7 @@ v2.0.0 (2015-05-12) their defaults. - (new) introduce a way to specify on which platform a testenvironment is to - execute: the new per-venv "platform" setting allows to specify + execute: the new per-venv "platform" setting allows one to specify a regular expression which is matched against sys.platform. If platform is set and doesn't match the platform spec in the test environment the test environment is ignored, no setup or tests are attempted. @@ -1173,7 +1173,7 @@ v2.0.0 (2015-05-12) - tox has now somewhat pep8 clean code, thanks to Volodymyr Vitvitski. -- fix `#240 `_: allow to specify empty argument list without it being +- fix `#240 `_: allow one to specify empty argument list without it being rewritten to ".". Thanks Daniel Hahler. - introduce experimental (not much documented yet) plugin system @@ -1310,7 +1310,7 @@ v1.7.0 (2014-01-29) support for creating python2.5 based environments anymore and all internal special-handling has been removed. -- merged PR81: new option --force-dep which allows to +- merged PR81: new option --force-dep which allows one to override tox.ini specified dependencies in setuptools-style. For example "--force-dep 'django<1.6'" will make sure that any environment using "django" as a dependency will @@ -1381,7 +1381,7 @@ v1.6.1 (2013-09-04) to install ssl and/or use PIP_INSECURE=1 through ``setenv``. section. - fix `#102 `_: change to {toxinidir} when installing dependencies. - this allows to use relative path like in "-rrequirements.txt". + This allows one to use relative path like in "-rrequirements.txt". v1.6.0 (2013-08-15) ------------------- @@ -1537,7 +1537,7 @@ v1.4 (2012-06-13) v1.3 2011-12-21 --------------- -- fix: allow to specify wildcard filesystem paths when +- fix: allow one to specify wildcard filesystem paths when specifying dependencies such that tox searches for the highest version diff --git a/docs/drafts/extend-envs-and-packagebuilds.md b/docs/drafts/extend-envs-and-packagebuilds.md index a4c73a193..cecdb37c4 100644 --- a/docs/drafts/extend-envs-and-packagebuilds.md +++ b/docs/drafts/extend-envs-and-packagebuilds.md @@ -70,7 +70,7 @@ one to one relationship from environment to directory ## Proposal -This feature shall allow to specify how plugins can specify new types of package formats and environments to run test +This feature shall allow one to specify how plugins can specify new types of package formats and environments to run test commands in. Such plugins would take care of setting up the environment, create packages and run test commands using hooks provided diff --git a/docs/example/general.rst b/docs/example/general.rst index d5532335c..10080d439 100644 --- a/docs/example/general.rst +++ b/docs/example/general.rst @@ -125,7 +125,7 @@ Access package artifacts between multiple tox-runs If you have multiple projects using tox you can make use of a ``distshare`` directory where ``tox`` will copy in sdist-packages so that another tox run can find the "latest" dependency. This feature -allows to test a package against an unreleased development version +allows you to test a package against an unreleased development version or even an uncommitted version on your own machine. By default, ``{homedir}/.tox/distshare`` will be used for diff --git a/docs/example/jenkins.rst b/docs/example/jenkins.rst index 81795128d..c5ca8a8c6 100644 --- a/docs/example/jenkins.rst +++ b/docs/example/jenkins.rst @@ -4,7 +4,7 @@ Using tox with the Jenkins Integration Server Using Jenkins multi-configuration jobs ------------------------------------------- -The Jenkins_ continuous integration server allows to define "jobs" with +The Jenkins_ continuous integration server allows you to define "jobs" with "build steps" which can be test invocations. If you :doc:`install <../install>` ``tox`` on your default Python installation on each Jenkins agent, you can easily create a Jenkins multi-configuration job that will drive your tox runs from the CI-server side, diff --git a/docs/example/nose.rst b/docs/example/nose.rst index 11637c270..bd4339e4a 100644 --- a/docs/example/nose.rst +++ b/docs/example/nose.rst @@ -21,7 +21,7 @@ and the following ``tox.ini`` content: [testenv] deps = nose - # ``{posargs}`` will be substituted with positional arguments from comand line + # ``{posargs}`` will be substituted with positional arguments from command line commands = nosetests {posargs} you can invoke ``tox`` in the directory where your ``tox.ini`` resides. diff --git a/docs/example/unittest.rst b/docs/example/unittest.rst index 2d43af5ab..adf0ccae8 100644 --- a/docs/example/unittest.rst +++ b/docs/example/unittest.rst @@ -4,8 +4,8 @@ unittest2, discover and tox Running unittests with 'discover' ------------------------------------------ -The discover_ project allows to discover and run unittests -and we can easily integrate it in a ``tox`` run. As an example, +The discover_ project allows you to discover and run unittests +that you can easily integrate it in a ``tox`` run. As an example, perform a checkout of `Pygments `_: .. code-block:: shell From 829cad6aee43bbed5d2a087e705680af39dd8579 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 1 Sep 2019 21:38:51 +0200 Subject: [PATCH 71/72] Add initial GitHub Actions CI/CD linters workflow --- .github/workflows/python-linters.yml | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/python-linters.yml diff --git a/.github/workflows/python-linters.yml b/.github/workflows/python-linters.yml new file mode 100644 index 000000000..f231d5d56 --- /dev/null +++ b/.github/workflows/python-linters.yml @@ -0,0 +1,46 @@ +name: Code quality + +on: + push: + pull_request: + schedule: + # Run every Friday at 18:02 UTC + # https://crontab.guru/#2_18_*_*_5 + - cron: 2 18 * * 5 + +jobs: + linters: + name: 🤖 + runs-on: ${{ matrix.os }} + strategy: + # max-parallel: 5 + matrix: + os: + - ubuntu-18.04 + python-version: + - 3.7 + env: + - TOXENV: fix_lint + - TOXENV: docs + - TOXENV: dev + - TOXENV: package_description + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + version: ${{ matrix.python-version }} + - name: Install tox + run: | + python -m pip install --upgrade tox + - name: 'Initialize tox envs: ${{ matrix.env.TOXENV }}' + run: >- + python + -m tox + --notest + --skip-missing-interpreters false + env: ${{ matrix.env }} + - name: Test with tox + run: >- + python -m tox + env: ${{ matrix.env }} From 7faa411cffd7c7b27f5deda8692bab373d8112cd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 1 Sep 2019 21:41:34 +0200 Subject: [PATCH 72/72] Add a change log note --- docs/changelog/1411.misc.gh-actions-ci-cd--linters.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changelog/1411.misc.gh-actions-ci-cd--linters.rst diff --git a/docs/changelog/1411.misc.gh-actions-ci-cd--linters.rst b/docs/changelog/1411.misc.gh-actions-ci-cd--linters.rst new file mode 100644 index 000000000..f7576f33b --- /dev/null +++ b/docs/changelog/1411.misc.gh-actions-ci-cd--linters.rst @@ -0,0 +1 @@ +Add GitHub Actions CI/CD workflow running linters — by :user:`webknjaz`