Merge pull request #4865 from blueyed/revisit-cov-new

ci: revisit coverage reporting

Author: blueyed

.travis.yml

@@ -10,45 +10,23 @@ python:
   - '3.7'
 install:
   - python -m pip install --upgrade --pre tox
-env:
-  matrix:
-    - TOXENV=py27 PYTEST_COVERAGE=1
-    # Specialized factors for py27.
-    - TOXENV=py27-nobyte-numpy-xdist PYTEST_COVERAGE=1
-    - TOXENV=py27-pluggymaster-xdist
-    # Specialized factors for py37.
-    - TOXENV=py37-pexpect,py37-trial
-    - TOXENV=py37-pluggymaster-xdist
-    - TOXENV=py37-freeze
-
-matrix:
-  allow_failures:
-    - python: '3.8-dev'
-      env: TOXENV=py38-xdist
 
 jobs:
   include:
-    # Coverage tracking is slow with pypy, skip it.
-    - env: TOXENV=pypy-xdist
-      python: 'pypy2.7-6.0'
-    - env: TOXENV=pypy3-xdist
-      python: 'pypy3.5-6.0'
-
-    - env: TOXENV=py34-xdist
-      python: '3.4'
-    - env: TOXENV=py35-xdist
-      python: '3.5'
-    - env: TOXENV=py36-xdist
-      python: '3.6'
-    - env: TOXENV=py37
+    # OSX tests - first (in test stage), since they are the slower ones.
     - &test-macos
+      # NOTE: (tests with) pexpect appear to be buggy on Travis,
+      #       at least with coverage.
+      #       Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864
       os: osx
       osx_image: xcode10.1
       language: generic
+      # Coverage for:
+      # - py2 with symlink in test_cmdline_python_package_symlink.
+      env: TOXENV=py27-xdist PYTEST_COVERAGE=1
       before_install:
         - python -V
         - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27
-      env: TOXENV=py27-xdist
     - <<: *test-macos
       env: TOXENV=py37-xdist
       before_install:
@@ -58,16 +36,56 @@ jobs:
         - python -V
         - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37
 
+    # Full run of latest (major) supported versions, without xdist.
+    - env: TOXENV=py27
+      python: '2.7'
+    - env: TOXENV=py37
+      python: '3.7'
+
+    # Coverage tracking is slow with pypy, skip it.
+    - env: TOXENV=pypy-xdist
+      python: 'pypy2.7-6.0'
+    - env: TOXENV=pypy3-xdist
+      python: 'pypy3.5-6.0'
+
+    - env: TOXENV=py34-xdist
+      python: '3.4'
+    - env: TOXENV=py35-xdist
+      python: '3.5'
+
+    # Coverage for:
+    # - pytester's LsofFdLeakChecker
+    # - TestArgComplete (linux only)
+    # - numpy
+    - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1
+
+    # Specialized factors for py27.
+    - env: TOXENV=py27-nobyte-numpy-xdist
+      python: '2.7'
+    - env: TOXENV=py27-pluggymaster-xdist
+      python: '2.7'
+
+    # Specialized factors for py37.
+    # Coverage for:
+    # - test_sys_breakpoint_interception (via pexpect).
+    - env: TOXENV=py37-pexpect,py37-trial PYTEST_COVERAGE=1
+    - env: TOXENV=py37-pluggymaster-xdist
+    - env: TOXENV=py37-freeze
+
     # Jobs only run via Travis cron jobs (currently daily).
     - env: TOXENV=py38-xdist
       python: '3.8-dev'
       if: type = cron
 
     - stage: baseline
-      env: TOXENV=py27-pexpect,py27-trial
-    - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1
-    - env: TOXENV=linting,docs,doctesting
-      python: '3.7'
+    # Coverage for:
+    # - _pytest.unittest._handle_skip (via pexpect).
+      env: TOXENV=py27-pexpect,py27-trial PYTEST_COVERAGE=1
+      python: '2.7'
+    # Use py36 here for faster baseline.
+    - env: TOXENV=py36-xdist
+      python: '3.6'
+    - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1
 
     - stage: deploy
       python: '3.6'
@@ -84,6 +102,11 @@ jobs:
           tags: true
           repo: pytest-dev/pytest
 
+matrix:
+  allow_failures:
+    - python: '3.8-dev'
+      env: TOXENV=py38-xdist
+
 before_script:
   - |
     # Do not (re-)upload coverage with cron runs.
@@ -109,7 +132,7 @@ after_success:
       coverage combine
       coverage xml
       coverage report -m
-      bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV
+      bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME
     fi
 
 notifications:

azure-pipelines.yml

@@ -20,19 +20,28 @@ jobs:
       py27:
         python.version: '2.7'
         tox.env: 'py27'
-        PYTEST_COVERAGE: '1'
-      py27-nobyte-numpy-xdist:
+      py27-nobyte-lsof-numpy:
         python.version: '2.7'
-        tox.env: 'py27-nobyte-numpy-xdist'
+        tox.env: 'py27-lsof-nobyte-numpy'
+        # Coverage for:
+        # - test_supports_breakpoint_module_global
+        # - test_terminal_reporter_writer_attr (without xdist)
+        # - "if write" branch in _pytest.assertion.rewrite
+        # - numpy
+        # - pytester's LsofFdLeakChecker (being skipped)
         PYTEST_COVERAGE: '1'
       py27-trial:
         python.version: '2.7'
         tox.env: 'py27-trial'
         python.needs_vc: True
-        PYTEST_COVERAGE: '1'
       py27-pluggymaster-xdist:
         python.version: '2.7'
         tox.env: 'py27-pluggymaster-xdist'
+        # Coverage for:
+        # - except-IOError in _attempt_to_close_capture_file for py2.
+        #   Also seen with py27-nobyte (using xdist), and py27-xdist.
+        #   But no exception with py27-pexpect,py27-trial,py27-numpy.
+        PYTEST_COVERAGE: '1'
       pypy:
         python.version: 'pypy'
         tox.env: 'pypy'
@@ -42,33 +51,34 @@ jobs:
       #   python.version: 'pypy3'
       #   tox.env: 'pypy3'
       #   python.exe: 'pypy3'
-      py34:
+      py34-xdist:
         python.version: '3.4'
-        tox.env: 'py34'
+        tox.env: 'py34-xdist'
+        # Coverage for:
+        # - _pytest.compat._bytes_to_ascii
         PYTEST_COVERAGE: '1'
-      py35:
+      py35-xdist:
         python.version: '3.5'
-        tox.env: 'py35'
+        tox.env: 'py35-xdist'
+        # Coverage for:
+        # - test_supports_breakpoint_module_global
         PYTEST_COVERAGE: '1'
-      py36:
+      py36-xdist:
         python.version: '3.6'
-        tox.env: 'py36'
-        PYTEST_COVERAGE: '1'
+        tox.env: 'py36-xdist'
       py37:
         python.version: '3.7'
         tox.env: 'py37'
+        # Coverage for:
+        # - _py36_windowsconsoleio_workaround (with py36+)
+        # - test_request_garbage (no xdist)
         PYTEST_COVERAGE: '1'
       py37-linting/docs/doctesting:
         python.version: '3.7'
         tox.env: 'linting,docs,doctesting'
-      py37-xdist:
-        python.version: '3.7'
-        tox.env: 'py37-xdist'
-        PYTEST_COVERAGE: '1'
       py37-trial/numpy:
         python.version: '3.7'
         tox.env: 'py37-trial,py37-numpy'
-        PYTEST_COVERAGE: '1'
       py37-pluggymaster-xdist:
         python.version: '3.7'
         tox.env: 'py37-pluggymaster-xdist'

testing/test_assertrewrite.py

@@ -1313,8 +1313,7 @@ def test_cwd_changed(self, testdir, monkeypatch):
         # always (previously triggered via xdist only).
         # Ref: https://github.com/pytest-dev/py/pull/207
         monkeypatch.setattr(sys, "path", [""] + sys.path)
-        if "pathlib" in sys.modules:
-            del sys.modules["pathlib"]
+        monkeypatch.delitem(sys.modules, "pathlib", raising=False)
 
         testdir.makepyfile(
             **{