ci: publish test reports (#2091)

Use the dorny/test-reporter GitHub Action to publish test reports for
both mypy and pytest.

This should make it easier to see what fails without having to dig
through the rather noisy console logs.

Author: aucampia

.github/workflows/test-report.yml

@@ -0,0 +1,27 @@
+name: "Test Report"
+on:
+  workflow_run:
+    workflows: ["Validate"]
+    types:
+      - completed
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      statuses: write
+    steps:
+      - uses: dorny/test-reporter@v1
+        with:
+          artifact: /(.*)-mypy-junit-xml$/
+          name: mypy report
+          path: "*.xml"
+          reporter: java-junit
+          fail-on-error: "false"
+      - uses: dorny/test-reporter@v1
+        with:
+          artifact: /(.*)-pytest-junit-xml$/
+          name: pytest report
+          path: "*.xml"
+          reporter: java-junit
+          fail-on-error: "false"

.github/workflows/validate.yaml

@@ -77,9 +77,20 @@ jobs:
             EXTENSIVE=${{ matrix.extensive-tests || 'false' }} \
             TOX_PYTHON_VERSION=${{ matrix.python-version }} \
             TOXENV_SUFFIX=${{ matrix.TOXENV_SUFFIX }} \
+            TOX_JUNIT_XML_PREFIX=${{ matrix.python-version }}-${{ matrix.os }}- \
             gha:validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/upload-artifact@v2
+        if: ${{ (success() || failure()) }}
+        with:
+          name: ${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-mypy-junit.xml
+      - uses: actions/upload-artifact@v2
+        if: ${{ (success() || failure()) }}
+        with:
+          name: ${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit-xml
+          path: test_reports/${{ matrix.python-version }}-${{ matrix.os }}-pytest-junit.xml
   extra-tasks:
     runs-on: ubuntu-latest
     strategy:

.gitignore

@@ -12,7 +12,8 @@ test_reports/*latest.ttl
 # PyCharm
 .idea/
 prepare_changelog.sh
-#### vimdiff <(curl --silent -L https://github.com/github/gitignore/raw/master/Python.gitignore) .gitignore
+*-junit.xml
+#### vimdiff <(curl --silent -L https://github.com/github/gitignore/raw/main/Python.gitignore) .gitignore
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -109,7 +110,22 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
 # Celery stuff
@@ -152,3 +168,9 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Taskfile.yml

@@ -120,6 +120,7 @@ tasks:
       - |
         {{if .TOX_PYTEST_ARGS}}TOX_PYTEST_ARGS={{shellQuote .TOX_PYTEST_ARGS}}{{end}} \
         {{if .TOX_EXTRA_COMMAND}}TOX_EXTRA_COMMAND={{shellQuote .TOX_EXTRA_COMMAND}}{{end}} \
+        {{if .TOX_JUNIT_XML_PREFIX}}TOX_JUNIT_XML_PREFIX={{shellQuote .TOX_JUNIT_XML_PREFIX}}{{end}} \
         {{if .COVERAGE_FILE}}COVERAGE_FILE={{shellQuote .COVERAGE_FILE}}{{end}} \
         {{.TEST_HARNESS}} \
         {{._PYTHON | shellQuote}} \

tox.ini

@@ -20,8 +20,8 @@ extras =
     docs: docs
 commands =
     {env:TOX_EXTRA_COMMAND:}
-    {env:TOX_MYPY_COMMAND:{envpython} -m mypy --show-error-context --show-error-codes}
-    {posargs:{envpython} -m pytest -ra --tb=native {env:TOX_PYTEST_ARGS:--cov --cov-report=}}
+    {env:TOX_MYPY_COMMAND:{envpython} -m mypy --show-error-context --show-error-codes --junit-xml=test_reports/{env:TOX_JUNIT_XML_PREFIX:}mypy-junit.xml}
+    {posargs:{envpython} -m pytest -ra --tb=native {env:TOX_PYTEST_ARGS:--junit-xml=test_reports/{env:TOX_JUNIT_XML_PREFIX:}pytest-junit.xml --cov --cov-report=}}
     docs: sphinx-build -n -T -W -b html -d {envdir}/doctree docs docs/_build/html
 
 [testenv:covreport]